{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import f1_score, recall_score, precision_score, classification_report\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import pandas as pd\n",
    "pd.options.mode.chained_assignment = None  # default='warn'\n",
    "import numpy as np\n",
    "\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Vocabulary storing how features should be modified at each step\n",
    "steps = {\n",
    "    1 : {\n",
    "        'Dur' : 1, 'SrcBytes' : 1, 'DstBytes' : 1, 'TotPkts' : 1 },\n",
    "    2 : {\n",
    "        'Dur' : 2, 'SrcBytes' : 2, 'DstBytes' : 2, 'TotPkts' : 2 },\n",
    "    3 : {\n",
    "        'Dur' : 5, 'SrcBytes' : 8, 'DstBytes' : 8, 'TotPkts' : 5 },\n",
    "    4 : {\n",
    "        'Dur' : 10, 'SrcBytes' : 16, 'DstBytes' : 16, 'TotPkts' : 10 },\n",
    "    5 : {\n",
    "        'Dur' : 15, 'SrcBytes' : 64, 'DstBytes' : 64, 'TotPkts' : 15 },\n",
    "    6 : {\n",
    "        'Dur' : 30, 'SrcBytes' : 128, 'DstBytes' : 128, 'TotPkts' : 20 },\n",
    "    7 : {\n",
    "        'Dur' : 45, 'SrcBytes' : 256, 'DstBytes' : 256, 'TotPkts' : 30 },\n",
    "    8 : {\n",
    "        'Dur' : 60, 'SrcBytes' : 512, 'DstBytes' : 512, 'TotPkts' : 50 },\n",
    "    9 : {\n",
    "        'Dur' : 120, 'SrcBytes' : 1024, 'DstBytes' : 1024, 'TotPkts' : 100 }\n",
    "}\n",
    "\n",
    "#Groups of altered features\n",
    "altered_features = {\n",
    "    '1a' : ['Dur'],\n",
    "    '1b' : ['SrcBytes'],\n",
    "    '1c' : ['DstBytes'],\n",
    "    '1d' : ['TotPkts'],\n",
    "    '2a' : ['Dur', 'SrcBytes'],\n",
    "    '2b' : ['Dur', 'DstBytes'],\n",
    "    '2c' : ['Dur', 'TotPkts'],\n",
    "    '2d' : ['SrcBytes', 'TotPkts'],\n",
    "    '2e' : ['SrcBytes', 'DstBytes'],\n",
    "    '2f' : ['DstBytes', 'TotPkts'],\n",
    "    '3a' : ['Dur', 'SrcBytes', 'DstBytes'],\n",
    "    '3b' : ['Dur', 'SrcBytes', 'TotPkts'],\n",
    "    '3c' : ['Dur', 'DstBytes', 'TotPkts'],\n",
    "    '3d' : ['SrcBytes','DstBytes','TotPkts'],\n",
    "    '4a' : ['Dur', 'SrcBytes', 'DstBytes', 'TotPkts']\n",
    "}\n",
    "\n",
    "#Function returning the modified dataset according to the list of features to alter\n",
    "def get_malicious_modified(feature_key, malicious, step):\n",
    "    for i in altered_features[feature_key]:\n",
    "        modified = malicious[i] + steps[step][i]\n",
    "        malicious.loc[:, i] = modified\n",
    "    return malicious\n",
    "\n",
    "def correct_derived_features(malicious, ton_iot=False):\n",
    "    malicious.loc[:, 'TotBytes'] = malicious['SrcBytes'] + malicious['DstBytes']\n",
    "    if not ton_iot: # ToN_IoT does not have these derived features\n",
    "        malicious.loc[:, 'BytesPerPkt'] = malicious['TotBytes'] / malicious['TotPkts']\n",
    "        malicious.loc[:, 'PktsPerSec'] = malicious['TotPkts'] / malicious['Dur']\n",
    "        max_value = malicious.loc[malicious['PktsPerSec'] != np.inf, 'PktsPerSec'].max()\n",
    "        malicious.loc[:, 'PktsPerSec'].replace(np.inf, max_value, inplace=True)\n",
    "        malicious.loc[:, 'RatioOutIn'] = (malicious['DstBytes']) / malicious['SrcBytes']\n",
    "        max_value = malicious.loc[malicious['RatioOutIn'] != np.inf, 'RatioOutIn'].max()\n",
    "        malicious.loc[:, 'RatioOutIn'].replace(np.inf, max_value, inplace=True)\n",
    "    return malicious\n",
    "\n",
    "def adversarial_feature_attack(test_data, model, ton_iot = False):\n",
    "    recall_dict = {}\n",
    "    precision_dict = {}\n",
    "    f1_dict = {}\n",
    "    \n",
    "    pbar = tqdm(range(len(altered_features.keys())*len(steps.keys())))\n",
    "    for k in altered_features.keys():\n",
    "        group_recalls = []\n",
    "        group_precisions = []\n",
    "        group_f1 = []\n",
    "        for s in steps.keys():\n",
    "            data = test_data.copy()\n",
    "            data[data.Label == 1] = get_malicious_modified(k, data[data.Label == 1], s)\n",
    "            data[data.Label == 1] = correct_derived_features(data[data.Label == 1], ton_iot)\n",
    "            \n",
    "            predictions = model.predict(data.drop(columns=['Label']))\n",
    "            report = classification_report(data.Label, predictions, digits=3, output_dict=True)\n",
    "            report = pd.DataFrame(report).transpose()\n",
    "            \n",
    "            recall = report.loc['1', 'recall']\n",
    "            precision = report.loc['1', 'precision']\n",
    "            f1 = report.loc['1', 'f1-score']\n",
    "            \n",
    "            group_recalls.append(recall)\n",
    "            group_precisions.append(precision)\n",
    "            group_f1.append(f1)\n",
    "            \n",
    "            pbar.write(f\"Group {k}, step {s}: {report.loc['1', 'recall']}\")\n",
    "            pbar.update(1)\n",
    "        \n",
    "        recall_dict[k] = group_recalls\n",
    "        precision_dict[k] = group_precisions\n",
    "        f1_dict[k] = group_f1\n",
    "        \n",
    "    recall_df = pd.DataFrame.from_dict(recall_dict)\n",
    "    precision_df = pd.DataFrame.from_dict(precision_dict)\n",
    "    f1_df = pd.DataFrame.from_dict(f1_dict)\n",
    "    \n",
    "    return recall_df.T, precision_df.T, f1_df.T\n",
    "            \n",
    "def test_adversarial_feature_attack():\n",
    "    data = pd.read_csv(\"preprocessed_data/murlo_test.csv\")\n",
    "    test_d = data.copy()\n",
    "    print(f\"Original Mal: {test_d.loc[test_d.Label == 1, ['Dur', 'SrcBytes', 'PktsPerSec']].head()}\")\n",
    "    print(f\"Original ben: {test_d.loc[test_d.Label == 0, ['Dur', 'SrcBytes', 'PktsPerSec']].head()}\")\n",
    "    \n",
    "    test_d[test_d.Label == 1] = get_malicious_modified('1a', test_d.loc[test_d.Label == 1], 9)\n",
    "    test_d[test_d.Label == 1] = correct_derived_features(test_d)\n",
    "    print(f\"Modified Mal: {test_d.loc[test_d.Label == 1, ['Dur', 'SrcBytes', 'PktsPerSec']].head()}\")\n",
    "    print(f\"Modified ben: {test_d.loc[test_d.Label == 0, ['Dur', 'SrcBytes', 'PktsPerSec']].head()}\")\n",
    "    print(f\"Original TEST DATA: {data.loc[data.Label == 1, ['Dur']].head()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.994479</td>\n",
       "      <td>0.996629</td>\n",
       "      <td>0.995553</td>\n",
       "      <td>216539.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.963161</td>\n",
       "      <td>0.940939</td>\n",
       "      <td>0.951920</td>\n",
       "      <td>20284.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.991859</td>\n",
       "      <td>0.991859</td>\n",
       "      <td>0.991859</td>\n",
       "      <td>0.991859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>macro avg</th>\n",
       "      <td>0.978820</td>\n",
       "      <td>0.968784</td>\n",
       "      <td>0.973737</td>\n",
       "      <td>236823.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weighted avg</th>\n",
       "      <td>0.991797</td>\n",
       "      <td>0.991859</td>\n",
       "      <td>0.991816</td>\n",
       "      <td>236823.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              precision    recall  f1-score        support\n",
       "0              0.994479  0.996629  0.995553  216539.000000\n",
       "1              0.963161  0.940939  0.951920   20284.000000\n",
       "accuracy       0.991859  0.991859  0.991859       0.991859\n",
       "macro avg      0.978820  0.968784  0.973737  236823.000000\n",
       "weighted avg   0.991797  0.991859  0.991816  236823.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2bbf9bae2b7b41a787424326bbf8a896",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/135 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Group 1a, step 1: 0.8563399723920331\n",
      "Group 1a, step 2: 0.7394991126010649\n",
      "Group 1a, step 3: 0.45060145927824885\n",
      "Group 1a, step 4: 0.8591993689607572\n",
      "Group 1a, step 5: 0.8142378229146126\n",
      "Group 1a, step 6: 0.7648392821928613\n",
      "Group 1a, step 7: 0.7463025044369946\n",
      "Group 1a, step 8: 0.7436896075724709\n",
      "Group 1a, step 9: 0.7310195227765727\n",
      "Group 1b, step 1: 0.4962532044961546\n",
      "Group 1b, step 2: 0.369453756655492\n",
      "Group 1b, step 3: 0.23959771248274503\n",
      "Group 1b, step 4: 0.22737132715440742\n",
      "Group 1b, step 5: 0.1797968842437389\n",
      "Group 1b, step 6: 0.14523762571484913\n",
      "Group 1b, step 7: 0.11945375665549202\n",
      "Group 1b, step 8: 0.08716229540524552\n",
      "Group 1b, step 9: 0.06349832380201144\n",
      "Group 1c, step 1: 0.39232892920528495\n",
      "Group 1c, step 2: 0.36250246499704203\n",
      "Group 1c, step 3: 0.2376257148491422\n",
      "Group 1c, step 4: 0.22623742851508577\n",
      "Group 1c, step 5: 0.20084795898244923\n",
      "Group 1c, step 6: 0.17767698678761584\n",
      "Group 1c, step 7: 0.14789982252021297\n",
      "Group 1c, step 8: 0.12152435417077499\n",
      "Group 1c, step 9: 0.09317688818773417\n",
      "Group 1d, step 1: 0.34810688227174125\n",
      "Group 1d, step 2: 0.3146322224413331\n",
      "Group 1d, step 3: 0.25108459869848154\n",
      "Group 1d, step 4: 0.22549792940248473\n",
      "Group 1d, step 5: 0.21760993886807337\n",
      "Group 1d, step 6: 0.20922894892526128\n",
      "Group 1d, step 7: 0.2037566554920134\n",
      "Group 1d, step 8: 0.18393807927430486\n",
      "Group 1d, step 9: 0.16052060737527116\n",
      "Group 2a, step 1: 0.4353184776178269\n",
      "Group 2a, step 2: 0.2727765726681128\n",
      "Group 2a, step 3: 0.14947742062709524\n",
      "Group 2a, step 4: 0.1371524354170775\n",
      "Group 2a, step 5: 0.08543679747584303\n",
      "Group 2a, step 6: 0.051370538355353976\n",
      "Group 2a, step 7: 0.037911654506014594\n",
      "Group 2a, step 8: 0.03436205876552948\n",
      "Group 2a, step 9: 0.03322816012620785\n",
      "Group 2b, step 1: 0.3174916190100572\n",
      "Group 2b, step 2: 0.2683888779333465\n",
      "Group 2b, step 3: 0.1431177282587261\n",
      "Group 2b, step 4: 0.1250739499112601\n",
      "Group 2b, step 5: 0.0984026819167817\n",
      "Group 2b, step 6: 0.06221652534016959\n",
      "Group 2b, step 7: 0.050088739893512126\n",
      "Group 2b, step 8: 0.051074738710313546\n",
      "Group 2b, step 9: 0.041362650364819564\n",
      "Group 2c, step 1: 0.2901301518438178\n",
      "Group 2c, step 2: 0.2759317688818773\n",
      "Group 2c, step 3: 0.24260500887398936\n",
      "Group 2c, step 4: 0.22160323407611912\n",
      "Group 2c, step 5: 0.21864523762571486\n",
      "Group 2c, step 6: 0.22323013212384146\n",
      "Group 2c, step 7: 0.21366594360086769\n",
      "Group 2c, step 8: 0.2098698481561822\n",
      "Group 2c, step 9: 0.18364227962926444\n",
      "Group 2d, step 1: 0.24526720567935317\n",
      "Group 2d, step 2: 0.19793926247288504\n",
      "Group 2d, step 3: 0.14129363044764345\n",
      "Group 2d, step 4: 0.1177775586669296\n",
      "Group 2d, step 5: 0.18738907513310984\n",
      "Group 2d, step 6: 0.17491619010057188\n",
      "Group 2d, step 7: 0.1645139025833169\n",
      "Group 2d, step 8: 0.1519424176690988\n",
      "Group 2d, step 9: 0.1310392427529087\n",
      "Group 2e, step 1: 0.37837704594754484\n",
      "Group 2e, step 2: 0.29688424373890754\n",
      "Group 2e, step 3: 0.2245119305856833\n",
      "Group 2e, step 4: 0.20898244922106093\n",
      "Group 2e, step 5: 0.1585979096825084\n",
      "Group 2e, step 6: 0.12753894695326365\n",
      "Group 2e, step 7: 0.10269177676986788\n",
      "Group 2e, step 8: 0.07567540918950898\n",
      "Group 2e, step 9: 0.04772234273318872\n",
      "Group 2f, step 1: 0.3104910274107671\n",
      "Group 2f, step 2: 0.27149477420627094\n",
      "Group 2f, step 3: 0.2229836324196411\n",
      "Group 2f, step 4: 0.202080457503451\n",
      "Group 2f, step 5: 0.1920725695129166\n",
      "Group 2f, step 6: 0.1797968842437389\n",
      "Group 2f, step 7: 0.18448037862354566\n",
      "Group 2f, step 8: 0.17989548412541906\n",
      "Group 2f, step 9: 0.11156576612108066\n",
      "Group 3a, step 1: 0.2931374482350621\n",
      "Group 3a, step 2: 0.16140800631039243\n",
      "Group 3a, step 3: 0.13365213961743247\n",
      "Group 3a, step 4: 0.1143758627489647\n",
      "Group 3a, step 5: 0.07740090711891146\n",
      "Group 3a, step 6: 0.04604614474462631\n",
      "Group 3a, step 7: 0.03441135870636955\n",
      "Group 3a, step 8: 0.03337605994872806\n",
      "Group 3a, step 9: 0.03633405639913232\n",
      "Group 3b, step 1: 0.27879116545060145\n",
      "Group 3b, step 2: 0.24738710313547624\n",
      "Group 3b, step 3: 0.20927824886610136\n",
      "Group 3b, step 4: 0.1886708735949517\n",
      "Group 3b, step 5: 0.18748767501479\n",
      "Group 3b, step 6: 0.18285348057582332\n",
      "Group 3b, step 7: 0.22860382567540918\n",
      "Group 3b, step 8: 0.20538355353973575\n",
      "Group 3b, step 9: 0.14499112601064879\n",
      "Group 3c, step 1: 0.2800729639124433\n",
      "Group 3c, step 2: 0.2442319069217117\n",
      "Group 3c, step 3: 0.21406034312758826\n",
      "Group 3c, step 4: 0.19281206862551764\n",
      "Group 3c, step 5: 0.19182606980871622\n",
      "Group 3c, step 6: 0.19483336619996056\n",
      "Group 3c, step 7: 0.18245908104910274\n",
      "Group 3c, step 8: 0.16697889962532045\n",
      "Group 3c, step 9: 0.14351212778544667\n",
      "Group 3d, step 1: 0.2411753105896273\n",
      "Group 3d, step 2: 0.1922204693354368\n",
      "Group 3d, step 3: 0.14015973180832184\n",
      "Group 3d, step 4: 0.11383356339972392\n",
      "Group 3d, step 5: 0.18206468152238217\n",
      "Group 3d, step 6: 0.18827647406823111\n",
      "Group 3d, step 7: 0.1444981266022481\n",
      "Group 3d, step 8: 0.13143364227962925\n",
      "Group 3d, step 9: 0.04752514296982844\n",
      "Group 4a, step 1: 0.27322027213567346\n",
      "Group 4a, step 2: 0.23575231709721947\n",
      "Group 4a, step 3: 0.20587655294813645\n",
      "Group 4a, step 4: 0.18793137448235062\n",
      "Group 4a, step 5: 0.18719187536974954\n",
      "Group 4a, step 6: 0.6622954052455137\n",
      "Group 4a, step 7: 0.16357720370735554\n",
      "Group 4a, step 8: 0.15322421613094064\n",
      "Group 4a, step 9: 0.14045553145336226\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1a</th>\n",
       "      <td>0.856340</td>\n",
       "      <td>0.739499</td>\n",
       "      <td>0.450601</td>\n",
       "      <td>0.859199</td>\n",
       "      <td>0.814238</td>\n",
       "      <td>0.764839</td>\n",
       "      <td>0.746303</td>\n",
       "      <td>0.743690</td>\n",
       "      <td>0.731020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1b</th>\n",
       "      <td>0.496253</td>\n",
       "      <td>0.369454</td>\n",
       "      <td>0.239598</td>\n",
       "      <td>0.227371</td>\n",
       "      <td>0.179797</td>\n",
       "      <td>0.145238</td>\n",
       "      <td>0.119454</td>\n",
       "      <td>0.087162</td>\n",
       "      <td>0.063498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1c</th>\n",
       "      <td>0.392329</td>\n",
       "      <td>0.362502</td>\n",
       "      <td>0.237626</td>\n",
       "      <td>0.226237</td>\n",
       "      <td>0.200848</td>\n",
       "      <td>0.177677</td>\n",
       "      <td>0.147900</td>\n",
       "      <td>0.121524</td>\n",
       "      <td>0.093177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1d</th>\n",
       "      <td>0.348107</td>\n",
       "      <td>0.314632</td>\n",
       "      <td>0.251085</td>\n",
       "      <td>0.225498</td>\n",
       "      <td>0.217610</td>\n",
       "      <td>0.209229</td>\n",
       "      <td>0.203757</td>\n",
       "      <td>0.183938</td>\n",
       "      <td>0.160521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2a</th>\n",
       "      <td>0.435318</td>\n",
       "      <td>0.272777</td>\n",
       "      <td>0.149477</td>\n",
       "      <td>0.137152</td>\n",
       "      <td>0.085437</td>\n",
       "      <td>0.051371</td>\n",
       "      <td>0.037912</td>\n",
       "      <td>0.034362</td>\n",
       "      <td>0.033228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2b</th>\n",
       "      <td>0.317492</td>\n",
       "      <td>0.268389</td>\n",
       "      <td>0.143118</td>\n",
       "      <td>0.125074</td>\n",
       "      <td>0.098403</td>\n",
       "      <td>0.062217</td>\n",
       "      <td>0.050089</td>\n",
       "      <td>0.051075</td>\n",
       "      <td>0.041363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2c</th>\n",
       "      <td>0.290130</td>\n",
       "      <td>0.275932</td>\n",
       "      <td>0.242605</td>\n",
       "      <td>0.221603</td>\n",
       "      <td>0.218645</td>\n",
       "      <td>0.223230</td>\n",
       "      <td>0.213666</td>\n",
       "      <td>0.209870</td>\n",
       "      <td>0.183642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2d</th>\n",
       "      <td>0.245267</td>\n",
       "      <td>0.197939</td>\n",
       "      <td>0.141294</td>\n",
       "      <td>0.117778</td>\n",
       "      <td>0.187389</td>\n",
       "      <td>0.174916</td>\n",
       "      <td>0.164514</td>\n",
       "      <td>0.151942</td>\n",
       "      <td>0.131039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2e</th>\n",
       "      <td>0.378377</td>\n",
       "      <td>0.296884</td>\n",
       "      <td>0.224512</td>\n",
       "      <td>0.208982</td>\n",
       "      <td>0.158598</td>\n",
       "      <td>0.127539</td>\n",
       "      <td>0.102692</td>\n",
       "      <td>0.075675</td>\n",
       "      <td>0.047722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2f</th>\n",
       "      <td>0.310491</td>\n",
       "      <td>0.271495</td>\n",
       "      <td>0.222984</td>\n",
       "      <td>0.202080</td>\n",
       "      <td>0.192073</td>\n",
       "      <td>0.179797</td>\n",
       "      <td>0.184480</td>\n",
       "      <td>0.179895</td>\n",
       "      <td>0.111566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3a</th>\n",
       "      <td>0.293137</td>\n",
       "      <td>0.161408</td>\n",
       "      <td>0.133652</td>\n",
       "      <td>0.114376</td>\n",
       "      <td>0.077401</td>\n",
       "      <td>0.046046</td>\n",
       "      <td>0.034411</td>\n",
       "      <td>0.033376</td>\n",
       "      <td>0.036334</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3b</th>\n",
       "      <td>0.278791</td>\n",
       "      <td>0.247387</td>\n",
       "      <td>0.209278</td>\n",
       "      <td>0.188671</td>\n",
       "      <td>0.187488</td>\n",
       "      <td>0.182853</td>\n",
       "      <td>0.228604</td>\n",
       "      <td>0.205384</td>\n",
       "      <td>0.144991</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3c</th>\n",
       "      <td>0.280073</td>\n",
       "      <td>0.244232</td>\n",
       "      <td>0.214060</td>\n",
       "      <td>0.192812</td>\n",
       "      <td>0.191826</td>\n",
       "      <td>0.194833</td>\n",
       "      <td>0.182459</td>\n",
       "      <td>0.166979</td>\n",
       "      <td>0.143512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3d</th>\n",
       "      <td>0.241175</td>\n",
       "      <td>0.192220</td>\n",
       "      <td>0.140160</td>\n",
       "      <td>0.113834</td>\n",
       "      <td>0.182065</td>\n",
       "      <td>0.188276</td>\n",
       "      <td>0.144498</td>\n",
       "      <td>0.131434</td>\n",
       "      <td>0.047525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4a</th>\n",
       "      <td>0.273220</td>\n",
       "      <td>0.235752</td>\n",
       "      <td>0.205877</td>\n",
       "      <td>0.187931</td>\n",
       "      <td>0.187192</td>\n",
       "      <td>0.662295</td>\n",
       "      <td>0.163577</td>\n",
       "      <td>0.153224</td>\n",
       "      <td>0.140456</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2         3         4         5         6  \\\n",
       "1a  0.856340  0.739499  0.450601  0.859199  0.814238  0.764839  0.746303   \n",
       "1b  0.496253  0.369454  0.239598  0.227371  0.179797  0.145238  0.119454   \n",
       "1c  0.392329  0.362502  0.237626  0.226237  0.200848  0.177677  0.147900   \n",
       "1d  0.348107  0.314632  0.251085  0.225498  0.217610  0.209229  0.203757   \n",
       "2a  0.435318  0.272777  0.149477  0.137152  0.085437  0.051371  0.037912   \n",
       "2b  0.317492  0.268389  0.143118  0.125074  0.098403  0.062217  0.050089   \n",
       "2c  0.290130  0.275932  0.242605  0.221603  0.218645  0.223230  0.213666   \n",
       "2d  0.245267  0.197939  0.141294  0.117778  0.187389  0.174916  0.164514   \n",
       "2e  0.378377  0.296884  0.224512  0.208982  0.158598  0.127539  0.102692   \n",
       "2f  0.310491  0.271495  0.222984  0.202080  0.192073  0.179797  0.184480   \n",
       "3a  0.293137  0.161408  0.133652  0.114376  0.077401  0.046046  0.034411   \n",
       "3b  0.278791  0.247387  0.209278  0.188671  0.187488  0.182853  0.228604   \n",
       "3c  0.280073  0.244232  0.214060  0.192812  0.191826  0.194833  0.182459   \n",
       "3d  0.241175  0.192220  0.140160  0.113834  0.182065  0.188276  0.144498   \n",
       "4a  0.273220  0.235752  0.205877  0.187931  0.187192  0.662295  0.163577   \n",
       "\n",
       "           7         8  \n",
       "1a  0.743690  0.731020  \n",
       "1b  0.087162  0.063498  \n",
       "1c  0.121524  0.093177  \n",
       "1d  0.183938  0.160521  \n",
       "2a  0.034362  0.033228  \n",
       "2b  0.051075  0.041363  \n",
       "2c  0.209870  0.183642  \n",
       "2d  0.151942  0.131039  \n",
       "2e  0.075675  0.047722  \n",
       "2f  0.179895  0.111566  \n",
       "3a  0.033376  0.036334  \n",
       "3b  0.205384  0.144991  \n",
       "3c  0.166979  0.143512  \n",
       "3d  0.131434  0.047525  \n",
       "4a  0.153224  0.140456  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.999682</td>\n",
       "      <td>0.999690</td>\n",
       "      <td>0.999686</td>\n",
       "      <td>125958.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.994328</td>\n",
       "      <td>0.994184</td>\n",
       "      <td>0.994256</td>\n",
       "      <td>6877.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.999405</td>\n",
       "      <td>0.999405</td>\n",
       "      <td>0.999405</td>\n",
       "      <td>0.999405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>macro avg</th>\n",
       "      <td>0.997005</td>\n",
       "      <td>0.996937</td>\n",
       "      <td>0.996971</td>\n",
       "      <td>132835.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weighted avg</th>\n",
       "      <td>0.999405</td>\n",
       "      <td>0.999405</td>\n",
       "      <td>0.999405</td>\n",
       "      <td>132835.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              precision    recall  f1-score        support\n",
       "0              0.999682  0.999690  0.999686  125958.000000\n",
       "1              0.994328  0.994184  0.994256    6877.000000\n",
       "accuracy       0.999405  0.999405  0.999405       0.999405\n",
       "macro avg      0.997005  0.996937  0.996971  132835.000000\n",
       "weighted avg   0.999405  0.999405  0.999405  132835.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4dda6ff81c9a42b8ba0f94c28e97d6b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/135 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Group 1a, step 1: 0.9716446124763705\n",
      "Group 1a, step 2: 0.9728079104260579\n",
      "Group 1a, step 3: 0.9748436818380107\n",
      "Group 1a, step 4: 0.9704813145266832\n",
      "Group 1a, step 5: 0.9703359022829722\n",
      "Group 1a, step 6: 0.9613203431728952\n",
      "Group 1a, step 7: 0.9611749309291843\n",
      "Group 1a, step 8: 0.9611749309291843\n",
      "Group 1a, step 9: 0.9611749309291843\n",
      "Group 1b, step 1: 0.9895303184528137\n",
      "Group 1b, step 2: 0.08157626872182638\n",
      "Group 1b, step 3: 0.01541369783335757\n",
      "Group 1b, step 4: 0.014250399883670205\n",
      "Group 1b, step 5: 0.0\n",
      "Group 1b, step 6: 0.0\n",
      "Group 1b, step 7: 0.0\n",
      "Group 1b, step 8: 0.0\n",
      "Group 1b, step 9: 0.0\n",
      "Group 1c, step 1: 0.9898211429402356\n",
      "Group 1c, step 2: 0.08288497891522466\n",
      "Group 1c, step 3: 0.06659880761960157\n",
      "Group 1c, step 4: 0.054820415879017016\n",
      "Group 1c, step 5: 0.06412679947651592\n",
      "Group 1c, step 6: 0.05380253017304057\n",
      "Group 1c, step 7: 0.05249381997964229\n",
      "Group 1c, step 8: 0.04958557510542388\n",
      "Group 1c, step 9: 0.034026465028355386\n",
      "Group 1d, step 1: 0.8992293151083322\n",
      "Group 1d, step 2: 0.8915224661916533\n",
      "Group 1d, step 3: 0.8995201395957539\n",
      "Group 1d, step 4: 0.8905045804856769\n",
      "Group 1d, step 5: 0.8851243274683729\n",
      "Group 1d, step 6: 0.8855605641995056\n",
      "Group 1d, step 7: 0.8833793805438418\n",
      "Group 1d, step 8: 0.18569143521884543\n",
      "Group 1d, step 9: 0.18569143521884543\n",
      "Group 2a, step 1: 0.9672822451650429\n",
      "Group 2a, step 2: 0.07983132179729534\n",
      "Group 2a, step 3: 0.008579322378944307\n",
      "Group 2a, step 4: 0.00014541224371092046\n",
      "Group 2a, step 5: 0.0\n",
      "Group 2a, step 6: 0.0\n",
      "Group 2a, step 7: 0.0\n",
      "Group 2a, step 8: 0.0\n",
      "Group 2a, step 9: 0.0\n",
      "Group 2b, step 1: 0.9680093063835975\n",
      "Group 2b, step 2: 0.08070379525956085\n",
      "Group 2b, step 3: 0.012069216228006397\n",
      "Group 2b, step 4: 0.0007270612185546024\n",
      "Group 2b, step 5: 0.00014541224371092046\n",
      "Group 2b, step 6: 0.00014541224371092046\n",
      "Group 2b, step 7: 0.0\n",
      "Group 2b, step 8: 0.0\n",
      "Group 2b, step 9: 0.0\n",
      "Group 2c, step 1: 0.09248218700014542\n",
      "Group 2c, step 2: 0.08361204013377926\n",
      "Group 2c, step 3: 0.0809946197469827\n",
      "Group 2c, step 4: 0.0738694198051476\n",
      "Group 2c, step 5: 0.06558092191362512\n",
      "Group 2c, step 6: 0.06587174640104697\n",
      "Group 2c, step 7: 0.06572633415733604\n",
      "Group 2c, step 8: 0.06543550966991421\n",
      "Group 2c, step 9: 0.06543550966991421\n",
      "Group 2d, step 1: 0.08084920750327178\n",
      "Group 2d, step 2: 0.07735931365420969\n",
      "Group 2d, step 3: 0.027773738548785808\n",
      "Group 2d, step 4: 0.017594881489021375\n",
      "Group 2d, step 5: 0.012069216228006397\n",
      "Group 2d, step 6: 0.002472008143085648\n",
      "Group 2d, step 7: 0.0002908244874218409\n",
      "Group 2d, step 8: 0.0\n",
      "Group 2d, step 9: 0.0\n",
      "Group 2e, step 1: 0.08273956667151375\n",
      "Group 2e, step 2: 0.07765013814163152\n",
      "Group 2e, step 3: 0.013959575396248364\n",
      "Group 2e, step 4: 0.010178857059764432\n",
      "Group 2e, step 5: 0.0\n",
      "Group 2e, step 6: 0.0\n",
      "Group 2e, step 7: 0.0\n",
      "Group 2e, step 8: 0.0\n",
      "Group 2e, step 9: 0.0\n",
      "Group 2f, step 1: 0.10571470117783917\n",
      "Group 2f, step 2: 0.08826523193252872\n",
      "Group 2f, step 3: 0.08593863603315399\n",
      "Group 2f, step 4: 0.08564781154573214\n",
      "Group 2f, step 5: 0.06848916678784353\n",
      "Group 2f, step 6: 0.06514468518249236\n",
      "Group 2f, step 7: 0.06543550966991421\n",
      "Group 2f, step 8: 0.06514468518249236\n",
      "Group 2f, step 9: 0.06456303620764868\n",
      "Group 3a, step 1: 0.08114003199069361\n",
      "Group 3a, step 2: 0.07575977897338956\n",
      "Group 3a, step 3: 0.008724734622655227\n",
      "Group 3a, step 4: 0.00014541224371092046\n",
      "Group 3a, step 5: 0.0\n",
      "Group 3a, step 6: 0.0\n",
      "Group 3a, step 7: 0.0\n",
      "Group 3a, step 8: 0.0\n",
      "Group 3a, step 9: 0.0\n",
      "Group 3b, step 1: 0.08332121564635743\n",
      "Group 3b, step 2: 0.08201250545295914\n",
      "Group 3b, step 3: 0.07299694634288208\n",
      "Group 3b, step 4: 0.0\n",
      "Group 3b, step 5: 0.0\n",
      "Group 3b, step 6: 0.00014541224371092046\n",
      "Group 3b, step 7: 0.0\n",
      "Group 3b, step 8: 0.0\n",
      "Group 3b, step 9: 0.0\n",
      "Group 3c, step 1: 0.08332121564635743\n",
      "Group 3c, step 2: 0.08186709320924822\n",
      "Group 3c, step 3: 0.07808637487276429\n",
      "Group 3c, step 4: 0.06529009742620329\n",
      "Group 3c, step 5: 0.06529009742620329\n",
      "Group 3c, step 6: 0.06499927293878145\n",
      "Group 3c, step 7: 0.06499927293878145\n",
      "Group 3c, step 8: 0.0004362367311327614\n",
      "Group 3c, step 9: 0.0\n",
      "Group 3d, step 1: 0.08521157481459939\n",
      "Group 3d, step 2: 0.0802675585284281\n",
      "Group 3d, step 3: 0.07837719936018613\n",
      "Group 3d, step 4: 0.07706848916678784\n",
      "Group 3d, step 5: 0.0023265958993747273\n",
      "Group 3d, step 6: 0.0002908244874218409\n",
      "Group 3d, step 7: 0.0\n",
      "Group 3d, step 8: 0.0\n",
      "Group 3d, step 9: 0.0\n",
      "Group 4a, step 1: 0.08215791769667007\n",
      "Group 4a, step 2: 0.08084920750327178\n",
      "Group 4a, step 3: 0.0724152973680384\n",
      "Group 4a, step 4: 0.0004362367311327614\n",
      "Group 4a, step 5: 0.0\n",
      "Group 4a, step 6: 0.0\n",
      "Group 4a, step 7: 0.0\n",
      "Group 4a, step 8: 0.0\n",
      "Group 4a, step 9: 0.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1a</th>\n",
       "      <td>0.971645</td>\n",
       "      <td>0.972808</td>\n",
       "      <td>0.974844</td>\n",
       "      <td>0.970481</td>\n",
       "      <td>0.970336</td>\n",
       "      <td>0.961320</td>\n",
       "      <td>0.961175</td>\n",
       "      <td>0.961175</td>\n",
       "      <td>0.961175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1b</th>\n",
       "      <td>0.989530</td>\n",
       "      <td>0.081576</td>\n",
       "      <td>0.015414</td>\n",
       "      <td>0.014250</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1c</th>\n",
       "      <td>0.989821</td>\n",
       "      <td>0.082885</td>\n",
       "      <td>0.066599</td>\n",
       "      <td>0.054820</td>\n",
       "      <td>0.064127</td>\n",
       "      <td>0.053803</td>\n",
       "      <td>0.052494</td>\n",
       "      <td>0.049586</td>\n",
       "      <td>0.034026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1d</th>\n",
       "      <td>0.899229</td>\n",
       "      <td>0.891522</td>\n",
       "      <td>0.899520</td>\n",
       "      <td>0.890505</td>\n",
       "      <td>0.885124</td>\n",
       "      <td>0.885561</td>\n",
       "      <td>0.883379</td>\n",
       "      <td>0.185691</td>\n",
       "      <td>0.185691</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2a</th>\n",
       "      <td>0.967282</td>\n",
       "      <td>0.079831</td>\n",
       "      <td>0.008579</td>\n",
       "      <td>0.000145</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2b</th>\n",
       "      <td>0.968009</td>\n",
       "      <td>0.080704</td>\n",
       "      <td>0.012069</td>\n",
       "      <td>0.000727</td>\n",
       "      <td>0.000145</td>\n",
       "      <td>0.000145</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2c</th>\n",
       "      <td>0.092482</td>\n",
       "      <td>0.083612</td>\n",
       "      <td>0.080995</td>\n",
       "      <td>0.073869</td>\n",
       "      <td>0.065581</td>\n",
       "      <td>0.065872</td>\n",
       "      <td>0.065726</td>\n",
       "      <td>0.065436</td>\n",
       "      <td>0.065436</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2d</th>\n",
       "      <td>0.080849</td>\n",
       "      <td>0.077359</td>\n",
       "      <td>0.027774</td>\n",
       "      <td>0.017595</td>\n",
       "      <td>0.012069</td>\n",
       "      <td>0.002472</td>\n",
       "      <td>0.000291</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2e</th>\n",
       "      <td>0.082740</td>\n",
       "      <td>0.077650</td>\n",
       "      <td>0.013960</td>\n",
       "      <td>0.010179</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2f</th>\n",
       "      <td>0.105715</td>\n",
       "      <td>0.088265</td>\n",
       "      <td>0.085939</td>\n",
       "      <td>0.085648</td>\n",
       "      <td>0.068489</td>\n",
       "      <td>0.065145</td>\n",
       "      <td>0.065436</td>\n",
       "      <td>0.065145</td>\n",
       "      <td>0.064563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3a</th>\n",
       "      <td>0.081140</td>\n",
       "      <td>0.075760</td>\n",
       "      <td>0.008725</td>\n",
       "      <td>0.000145</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3b</th>\n",
       "      <td>0.083321</td>\n",
       "      <td>0.082013</td>\n",
       "      <td>0.072997</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000145</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3c</th>\n",
       "      <td>0.083321</td>\n",
       "      <td>0.081867</td>\n",
       "      <td>0.078086</td>\n",
       "      <td>0.065290</td>\n",
       "      <td>0.065290</td>\n",
       "      <td>0.064999</td>\n",
       "      <td>0.064999</td>\n",
       "      <td>0.000436</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3d</th>\n",
       "      <td>0.085212</td>\n",
       "      <td>0.080268</td>\n",
       "      <td>0.078377</td>\n",
       "      <td>0.077068</td>\n",
       "      <td>0.002327</td>\n",
       "      <td>0.000291</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4a</th>\n",
       "      <td>0.082158</td>\n",
       "      <td>0.080849</td>\n",
       "      <td>0.072415</td>\n",
       "      <td>0.000436</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2         3         4         5         6  \\\n",
       "1a  0.971645  0.972808  0.974844  0.970481  0.970336  0.961320  0.961175   \n",
       "1b  0.989530  0.081576  0.015414  0.014250  0.000000  0.000000  0.000000   \n",
       "1c  0.989821  0.082885  0.066599  0.054820  0.064127  0.053803  0.052494   \n",
       "1d  0.899229  0.891522  0.899520  0.890505  0.885124  0.885561  0.883379   \n",
       "2a  0.967282  0.079831  0.008579  0.000145  0.000000  0.000000  0.000000   \n",
       "2b  0.968009  0.080704  0.012069  0.000727  0.000145  0.000145  0.000000   \n",
       "2c  0.092482  0.083612  0.080995  0.073869  0.065581  0.065872  0.065726   \n",
       "2d  0.080849  0.077359  0.027774  0.017595  0.012069  0.002472  0.000291   \n",
       "2e  0.082740  0.077650  0.013960  0.010179  0.000000  0.000000  0.000000   \n",
       "2f  0.105715  0.088265  0.085939  0.085648  0.068489  0.065145  0.065436   \n",
       "3a  0.081140  0.075760  0.008725  0.000145  0.000000  0.000000  0.000000   \n",
       "3b  0.083321  0.082013  0.072997  0.000000  0.000000  0.000145  0.000000   \n",
       "3c  0.083321  0.081867  0.078086  0.065290  0.065290  0.064999  0.064999   \n",
       "3d  0.085212  0.080268  0.078377  0.077068  0.002327  0.000291  0.000000   \n",
       "4a  0.082158  0.080849  0.072415  0.000436  0.000000  0.000000  0.000000   \n",
       "\n",
       "           7         8  \n",
       "1a  0.961175  0.961175  \n",
       "1b  0.000000  0.000000  \n",
       "1c  0.049586  0.034026  \n",
       "1d  0.185691  0.185691  \n",
       "2a  0.000000  0.000000  \n",
       "2b  0.000000  0.000000  \n",
       "2c  0.065436  0.065436  \n",
       "2d  0.000000  0.000000  \n",
       "2e  0.000000  0.000000  \n",
       "2f  0.065145  0.064563  \n",
       "3a  0.000000  0.000000  \n",
       "3b  0.000000  0.000000  \n",
       "3c  0.000436  0.000000  \n",
       "3d  0.000000  0.000000  \n",
       "4a  0.000000  0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.998083</td>\n",
       "      <td>0.999332</td>\n",
       "      <td>0.998707</td>\n",
       "      <td>35946.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.986301</td>\n",
       "      <td>0.961603</td>\n",
       "      <td>0.973795</td>\n",
       "      <td>1797.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.997536</td>\n",
       "      <td>0.997536</td>\n",
       "      <td>0.997536</td>\n",
       "      <td>0.997536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>macro avg</th>\n",
       "      <td>0.992192</td>\n",
       "      <td>0.980468</td>\n",
       "      <td>0.986251</td>\n",
       "      <td>37743.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weighted avg</th>\n",
       "      <td>0.997522</td>\n",
       "      <td>0.997536</td>\n",
       "      <td>0.997521</td>\n",
       "      <td>37743.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              precision    recall  f1-score       support\n",
       "0              0.998083  0.999332  0.998707  35946.000000\n",
       "1              0.986301  0.961603  0.973795   1797.000000\n",
       "accuracy       0.997536  0.997536  0.997536      0.997536\n",
       "macro avg      0.992192  0.980468  0.986251  37743.000000\n",
       "weighted avg   0.997522  0.997536  0.997521  37743.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8801cdd62ca94b61adec1825baa21e1a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/135 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Group 1a, step 1: 0.7340011129660545\n",
      "Group 1a, step 2: 0.7212020033388982\n",
      "Group 1a, step 3: 0.7184195882025598\n",
      "Group 1a, step 4: 0.7167501391207568\n",
      "Group 1a, step 5: 0.7078464106844741\n",
      "Group 1a, step 6: 0.6989426822481914\n",
      "Group 1a, step 7: 0.6922648859209795\n",
      "Group 1a, step 8: 0.6905954368391765\n",
      "Group 1a, step 9: 0.6889259877573733\n",
      "Group 1b, step 1: 0.9565943238731218\n",
      "Group 1b, step 2: 0.7412353923205343\n",
      "Group 1b, step 3: 0.06900389538119088\n",
      "Group 1b, step 4: 0.06566499721758487\n",
      "Group 1b, step 5: 0.053978853644963826\n",
      "Group 1b, step 6: 0.031163049526989426\n",
      "Group 1b, step 7: 0.02615470228158041\n",
      "Group 1b, step 8: 0.018363939899833055\n",
      "Group 1b, step 9: 0.010573177518085699\n",
      "Group 1c, step 1: 0.9560378408458542\n",
      "Group 1c, step 2: 0.7406789092932665\n",
      "Group 1c, step 3: 0.06900389538119088\n",
      "Group 1c, step 4: 0.0651085141903172\n",
      "Group 1c, step 5: 0.06065664997217585\n",
      "Group 1c, step 6: 0.04897050639955482\n",
      "Group 1c, step 7: 0.038397328881469114\n",
      "Group 1c, step 8: 0.031163049526989426\n",
      "Group 1c, step 9: 0.02671118530884808\n",
      "Group 1d, step 1: 0.9232053422370617\n",
      "Group 1d, step 2: 0.32776850306065664\n",
      "Group 1d, step 3: 0.12520868113522537\n",
      "Group 1d, step 4: 0.20311630495269895\n",
      "Group 1d, step 5: 0.2025598219254313\n",
      "Group 1d, step 6: 0.19532554257095158\n",
      "Group 1d, step 7: 0.1914301613800779\n",
      "Group 1d, step 8: 0.18697829716193656\n",
      "Group 1d, step 9: 0.047857540345019475\n",
      "Group 2a, step 1: 0.7189760712298275\n",
      "Group 2a, step 2: 0.7184195882025598\n",
      "Group 2a, step 3: 0.044518642181413465\n",
      "Group 2a, step 4: 0.042292710072342796\n",
      "Group 2a, step 5: 0.02671118530884808\n",
      "Group 2a, step 6: 0.016138007790762382\n",
      "Group 2a, step 7: 0.010573177518085699\n",
      "Group 2a, step 8: 0.010573177518085699\n",
      "Group 2a, step 9: 0.009460211463550361\n",
      "Group 2b, step 1: 0.7189760712298275\n",
      "Group 2b, step 2: 0.7178631051752922\n",
      "Group 2b, step 3: 0.04340567612687813\n",
      "Group 2b, step 4: 0.041736227045075125\n",
      "Group 2b, step 5: 0.02949360044518642\n",
      "Group 2b, step 6: 0.01669449081803005\n",
      "Group 2b, step 7: 0.012242626599888704\n",
      "Group 2b, step 8: 0.01001669449081803\n",
      "Group 2b, step 9: 0.009460211463550361\n",
      "Group 2c, step 1: 0.7167501391207568\n",
      "Group 2c, step 2: 0.7161936560934892\n",
      "Group 2c, step 3: 0.7045075125208681\n",
      "Group 2c, step 4: 0.6961602671118531\n",
      "Group 2c, step 5: 0.6933778519755147\n",
      "Group 2c, step 6: 0.6878130217028381\n",
      "Group 2c, step 7: 0.6878130217028381\n",
      "Group 2c, step 8: 0.6872565386755703\n",
      "Group 2c, step 9: 0.6855870895937674\n",
      "Group 2d, step 1: 0.9198664440734557\n",
      "Group 2d, step 2: 0.048414023372287146\n",
      "Group 2d, step 3: 0.03005008347245409\n",
      "Group 2d, step 4: 0.022815804117974403\n",
      "Group 2d, step 5: 0.021146355036171398\n",
      "Group 2d, step 6: 0.017807456872565387\n",
      "Group 2d, step 7: 0.01669449081803005\n",
      "Group 2d, step 8: 0.014468558708959377\n",
      "Group 2d, step 9: 0.016138007790762382\n",
      "Group 2e, step 1: 0.7412353923205343\n",
      "Group 2e, step 2: 0.07401224262659989\n",
      "Group 2e, step 3: 0.0667779632721202\n",
      "Group 2e, step 4: 0.06399554813578186\n",
      "Group 2e, step 5: 0.05286588759042849\n",
      "Group 2e, step 6: 0.028380634390651086\n",
      "Group 2e, step 7: 0.021146355036171398\n",
      "Group 2e, step 8: 0.015025041736227046\n",
      "Group 2e, step 9: 0.011129660545353366\n",
      "Group 2f, step 1: 0.27545909849749584\n",
      "Group 2f, step 2: 0.14913745130773512\n",
      "Group 2f, step 3: 0.06176961602671119\n",
      "Group 2f, step 4: 0.05564830272676683\n",
      "Group 2f, step 5: 0.04897050639955482\n",
      "Group 2f, step 6: 0.047857540345019475\n",
      "Group 2f, step 7: 0.015581524763494713\n",
      "Group 2f, step 8: 0.012799109627156371\n",
      "Group 2f, step 9: 0.01001669449081803\n",
      "Group 3a, step 1: 0.7189760712298275\n",
      "Group 3a, step 2: 0.05008347245409015\n",
      "Group 3a, step 3: 0.0439621591541458\n",
      "Group 3a, step 4: 0.03672787979966611\n",
      "Group 3a, step 5: 0.02448525319977741\n",
      "Group 3a, step 6: 0.012799109627156371\n",
      "Group 3a, step 7: 0.012242626599888704\n",
      "Group 3a, step 8: 0.01001669449081803\n",
      "Group 3a, step 9: 0.009460211463550361\n",
      "Group 3b, step 1: 0.7161936560934892\n",
      "Group 3b, step 2: 0.044518642181413465\n",
      "Group 3b, step 3: 0.028937117417918753\n",
      "Group 3b, step 4: 0.021702838063439065\n",
      "Group 3b, step 5: 0.018920422927100722\n",
      "Group 3b, step 6: 0.015025041736227046\n",
      "Group 3b, step 7: 0.015025041736227046\n",
      "Group 3b, step 8: 0.011686143572621035\n",
      "Group 3b, step 9: 0.009460211463550361\n",
      "Group 3c, step 1: 0.7161936560934892\n",
      "Group 3c, step 2: 0.04062326099053979\n",
      "Group 3c, step 3: 0.031163049526989426\n",
      "Group 3c, step 4: 0.02448525319977741\n",
      "Group 3c, step 5: 0.022259321090706732\n",
      "Group 3c, step 6: 0.015025041736227046\n",
      "Group 3c, step 7: 0.012242626599888704\n",
      "Group 3c, step 8: 0.010573177518085699\n",
      "Group 3c, step 9: 0.009460211463550361\n",
      "Group 3d, step 1: 0.16805787423483584\n",
      "Group 3d, step 2: 0.0439621591541458\n",
      "Group 3d, step 3: 0.025598219254312743\n",
      "Group 3d, step 4: 0.022815804117974403\n",
      "Group 3d, step 5: 0.02003338898163606\n",
      "Group 3d, step 6: 0.016138007790762382\n",
      "Group 3d, step 7: 0.014468558708959377\n",
      "Group 3d, step 8: 0.012799109627156371\n",
      "Group 3d, step 9: 0.015025041736227046\n",
      "Group 4a, step 1: 0.7161936560934892\n",
      "Group 4a, step 2: 0.04006677796327212\n",
      "Group 4a, step 3: 0.02615470228158041\n",
      "Group 4a, step 4: 0.021146355036171398\n",
      "Group 4a, step 5: 0.016138007790762382\n",
      "Group 4a, step 6: 0.01335559265442404\n",
      "Group 4a, step 7: 0.011686143572621035\n",
      "Group 4a, step 8: 0.010573177518085699\n",
      "Group 4a, step 9: 0.009460211463550361\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1a</th>\n",
       "      <td>0.734001</td>\n",
       "      <td>0.721202</td>\n",
       "      <td>0.718420</td>\n",
       "      <td>0.716750</td>\n",
       "      <td>0.707846</td>\n",
       "      <td>0.698943</td>\n",
       "      <td>0.692265</td>\n",
       "      <td>0.690595</td>\n",
       "      <td>0.688926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1b</th>\n",
       "      <td>0.956594</td>\n",
       "      <td>0.741235</td>\n",
       "      <td>0.069004</td>\n",
       "      <td>0.065665</td>\n",
       "      <td>0.053979</td>\n",
       "      <td>0.031163</td>\n",
       "      <td>0.026155</td>\n",
       "      <td>0.018364</td>\n",
       "      <td>0.010573</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1c</th>\n",
       "      <td>0.956038</td>\n",
       "      <td>0.740679</td>\n",
       "      <td>0.069004</td>\n",
       "      <td>0.065109</td>\n",
       "      <td>0.060657</td>\n",
       "      <td>0.048971</td>\n",
       "      <td>0.038397</td>\n",
       "      <td>0.031163</td>\n",
       "      <td>0.026711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1d</th>\n",
       "      <td>0.923205</td>\n",
       "      <td>0.327769</td>\n",
       "      <td>0.125209</td>\n",
       "      <td>0.203116</td>\n",
       "      <td>0.202560</td>\n",
       "      <td>0.195326</td>\n",
       "      <td>0.191430</td>\n",
       "      <td>0.186978</td>\n",
       "      <td>0.047858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2a</th>\n",
       "      <td>0.718976</td>\n",
       "      <td>0.718420</td>\n",
       "      <td>0.044519</td>\n",
       "      <td>0.042293</td>\n",
       "      <td>0.026711</td>\n",
       "      <td>0.016138</td>\n",
       "      <td>0.010573</td>\n",
       "      <td>0.010573</td>\n",
       "      <td>0.009460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2b</th>\n",
       "      <td>0.718976</td>\n",
       "      <td>0.717863</td>\n",
       "      <td>0.043406</td>\n",
       "      <td>0.041736</td>\n",
       "      <td>0.029494</td>\n",
       "      <td>0.016694</td>\n",
       "      <td>0.012243</td>\n",
       "      <td>0.010017</td>\n",
       "      <td>0.009460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2c</th>\n",
       "      <td>0.716750</td>\n",
       "      <td>0.716194</td>\n",
       "      <td>0.704508</td>\n",
       "      <td>0.696160</td>\n",
       "      <td>0.693378</td>\n",
       "      <td>0.687813</td>\n",
       "      <td>0.687813</td>\n",
       "      <td>0.687257</td>\n",
       "      <td>0.685587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2d</th>\n",
       "      <td>0.919866</td>\n",
       "      <td>0.048414</td>\n",
       "      <td>0.030050</td>\n",
       "      <td>0.022816</td>\n",
       "      <td>0.021146</td>\n",
       "      <td>0.017807</td>\n",
       "      <td>0.016694</td>\n",
       "      <td>0.014469</td>\n",
       "      <td>0.016138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2e</th>\n",
       "      <td>0.741235</td>\n",
       "      <td>0.074012</td>\n",
       "      <td>0.066778</td>\n",
       "      <td>0.063996</td>\n",
       "      <td>0.052866</td>\n",
       "      <td>0.028381</td>\n",
       "      <td>0.021146</td>\n",
       "      <td>0.015025</td>\n",
       "      <td>0.011130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2f</th>\n",
       "      <td>0.275459</td>\n",
       "      <td>0.149137</td>\n",
       "      <td>0.061770</td>\n",
       "      <td>0.055648</td>\n",
       "      <td>0.048971</td>\n",
       "      <td>0.047858</td>\n",
       "      <td>0.015582</td>\n",
       "      <td>0.012799</td>\n",
       "      <td>0.010017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3a</th>\n",
       "      <td>0.718976</td>\n",
       "      <td>0.050083</td>\n",
       "      <td>0.043962</td>\n",
       "      <td>0.036728</td>\n",
       "      <td>0.024485</td>\n",
       "      <td>0.012799</td>\n",
       "      <td>0.012243</td>\n",
       "      <td>0.010017</td>\n",
       "      <td>0.009460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3b</th>\n",
       "      <td>0.716194</td>\n",
       "      <td>0.044519</td>\n",
       "      <td>0.028937</td>\n",
       "      <td>0.021703</td>\n",
       "      <td>0.018920</td>\n",
       "      <td>0.015025</td>\n",
       "      <td>0.015025</td>\n",
       "      <td>0.011686</td>\n",
       "      <td>0.009460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3c</th>\n",
       "      <td>0.716194</td>\n",
       "      <td>0.040623</td>\n",
       "      <td>0.031163</td>\n",
       "      <td>0.024485</td>\n",
       "      <td>0.022259</td>\n",
       "      <td>0.015025</td>\n",
       "      <td>0.012243</td>\n",
       "      <td>0.010573</td>\n",
       "      <td>0.009460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3d</th>\n",
       "      <td>0.168058</td>\n",
       "      <td>0.043962</td>\n",
       "      <td>0.025598</td>\n",
       "      <td>0.022816</td>\n",
       "      <td>0.020033</td>\n",
       "      <td>0.016138</td>\n",
       "      <td>0.014469</td>\n",
       "      <td>0.012799</td>\n",
       "      <td>0.015025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4a</th>\n",
       "      <td>0.716194</td>\n",
       "      <td>0.040067</td>\n",
       "      <td>0.026155</td>\n",
       "      <td>0.021146</td>\n",
       "      <td>0.016138</td>\n",
       "      <td>0.013356</td>\n",
       "      <td>0.011686</td>\n",
       "      <td>0.010573</td>\n",
       "      <td>0.009460</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2         3         4         5         6  \\\n",
       "1a  0.734001  0.721202  0.718420  0.716750  0.707846  0.698943  0.692265   \n",
       "1b  0.956594  0.741235  0.069004  0.065665  0.053979  0.031163  0.026155   \n",
       "1c  0.956038  0.740679  0.069004  0.065109  0.060657  0.048971  0.038397   \n",
       "1d  0.923205  0.327769  0.125209  0.203116  0.202560  0.195326  0.191430   \n",
       "2a  0.718976  0.718420  0.044519  0.042293  0.026711  0.016138  0.010573   \n",
       "2b  0.718976  0.717863  0.043406  0.041736  0.029494  0.016694  0.012243   \n",
       "2c  0.716750  0.716194  0.704508  0.696160  0.693378  0.687813  0.687813   \n",
       "2d  0.919866  0.048414  0.030050  0.022816  0.021146  0.017807  0.016694   \n",
       "2e  0.741235  0.074012  0.066778  0.063996  0.052866  0.028381  0.021146   \n",
       "2f  0.275459  0.149137  0.061770  0.055648  0.048971  0.047858  0.015582   \n",
       "3a  0.718976  0.050083  0.043962  0.036728  0.024485  0.012799  0.012243   \n",
       "3b  0.716194  0.044519  0.028937  0.021703  0.018920  0.015025  0.015025   \n",
       "3c  0.716194  0.040623  0.031163  0.024485  0.022259  0.015025  0.012243   \n",
       "3d  0.168058  0.043962  0.025598  0.022816  0.020033  0.016138  0.014469   \n",
       "4a  0.716194  0.040067  0.026155  0.021146  0.016138  0.013356  0.011686   \n",
       "\n",
       "           7         8  \n",
       "1a  0.690595  0.688926  \n",
       "1b  0.018364  0.010573  \n",
       "1c  0.031163  0.026711  \n",
       "1d  0.186978  0.047858  \n",
       "2a  0.010573  0.009460  \n",
       "2b  0.010017  0.009460  \n",
       "2c  0.687257  0.685587  \n",
       "2d  0.014469  0.016138  \n",
       "2e  0.015025  0.011130  \n",
       "2f  0.012799  0.010017  \n",
       "3a  0.010017  0.009460  \n",
       "3b  0.011686  0.009460  \n",
       "3c  0.010573  0.009460  \n",
       "3d  0.012799  0.015025  \n",
       "4a  0.010573  0.009460  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.999755</td>\n",
       "      <td>0.999755</td>\n",
       "      <td>0.999755</td>\n",
       "      <td>4075.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.995098</td>\n",
       "      <td>0.995098</td>\n",
       "      <td>0.995098</td>\n",
       "      <td>204.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.999533</td>\n",
       "      <td>0.999533</td>\n",
       "      <td>0.999533</td>\n",
       "      <td>0.999533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>macro avg</th>\n",
       "      <td>0.997426</td>\n",
       "      <td>0.997426</td>\n",
       "      <td>0.997426</td>\n",
       "      <td>4279.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weighted avg</th>\n",
       "      <td>0.999533</td>\n",
       "      <td>0.999533</td>\n",
       "      <td>0.999533</td>\n",
       "      <td>4279.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              precision    recall  f1-score      support\n",
       "0              0.999755  0.999755  0.999755  4075.000000\n",
       "1              0.995098  0.995098  0.995098   204.000000\n",
       "accuracy       0.999533  0.999533  0.999533     0.999533\n",
       "macro avg      0.997426  0.997426  0.997426  4279.000000\n",
       "weighted avg   0.999533  0.999533  0.999533  4279.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a71e0e11dfe04026b0db5fe628b53f63",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/135 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Group 1a, step 1: 0.9901960784313726\n",
      "Group 1a, step 2: 0.9901960784313726\n",
      "Group 1a, step 3: 0.9901960784313726\n",
      "Group 1a, step 4: 0.9901960784313726\n",
      "Group 1a, step 5: 0.9901960784313726\n",
      "Group 1a, step 6: 0.9901960784313726\n",
      "Group 1a, step 7: 0.9901960784313726\n",
      "Group 1a, step 8: 0.9901960784313726\n",
      "Group 1a, step 9: 0.9901960784313726\n",
      "Group 1b, step 1: 0.9950980392156863\n",
      "Group 1b, step 2: 0.9901960784313726\n",
      "Group 1b, step 3: 0.0392156862745098\n",
      "Group 1b, step 4: 0.0392156862745098\n",
      "Group 1b, step 5: 0.0392156862745098\n",
      "Group 1b, step 6: 0.0\n",
      "Group 1b, step 7: 0.0\n",
      "Group 1b, step 8: 0.0\n",
      "Group 1b, step 9: 0.0\n",
      "Group 1c, step 1: 0.9950980392156863\n",
      "Group 1c, step 2: 0.9852941176470589\n",
      "Group 1c, step 3: 0.0392156862745098\n",
      "Group 1c, step 4: 0.0392156862745098\n",
      "Group 1c, step 5: 0.0392156862745098\n",
      "Group 1c, step 6: 0.0392156862745098\n",
      "Group 1c, step 7: 0.0392156862745098\n",
      "Group 1c, step 8: 0.0392156862745098\n",
      "Group 1c, step 9: 0.0392156862745098\n",
      "Group 1d, step 1: 1.0\n",
      "Group 1d, step 2: 1.0\n",
      "Group 1d, step 3: 1.0\n",
      "Group 1d, step 4: 1.0\n",
      "Group 1d, step 5: 1.0\n",
      "Group 1d, step 6: 1.0\n",
      "Group 1d, step 7: 1.0\n",
      "Group 1d, step 8: 1.0\n",
      "Group 1d, step 9: 1.0\n",
      "Group 2a, step 1: 0.9901960784313726\n",
      "Group 2a, step 2: 0.9901960784313726\n",
      "Group 2a, step 3: 0.0392156862745098\n",
      "Group 2a, step 4: 0.0\n",
      "Group 2a, step 5: 0.0\n",
      "Group 2a, step 6: 0.0\n",
      "Group 2a, step 7: 0.0\n",
      "Group 2a, step 8: 0.0\n",
      "Group 2a, step 9: 0.0\n",
      "Group 2b, step 1: 0.9901960784313726\n",
      "Group 2b, step 2: 0.9901960784313726\n",
      "Group 2b, step 3: 0.0392156862745098\n",
      "Group 2b, step 4: 0.0392156862745098\n",
      "Group 2b, step 5: 0.0392156862745098\n",
      "Group 2b, step 6: 0.0392156862745098\n",
      "Group 2b, step 7: 0.004901960784313725\n",
      "Group 2b, step 8: 0.0\n",
      "Group 2b, step 9: 0.0\n",
      "Group 2c, step 1: 0.9901960784313726\n",
      "Group 2c, step 2: 0.9901960784313726\n",
      "Group 2c, step 3: 0.9901960784313726\n",
      "Group 2c, step 4: 0.9509803921568627\n",
      "Group 2c, step 5: 0.6617647058823529\n",
      "Group 2c, step 6: 0.9509803921568627\n",
      "Group 2c, step 7: 0.9509803921568627\n",
      "Group 2c, step 8: 0.6617647058823529\n",
      "Group 2c, step 9: 0.6617647058823529\n",
      "Group 2d, step 1: 1.0\n",
      "Group 2d, step 2: 1.0\n",
      "Group 2d, step 3: 0.6568627450980392\n",
      "Group 2d, step 4: 0.6568627450980392\n",
      "Group 2d, step 5: 0.6617647058823529\n",
      "Group 2d, step 6: 0.0\n",
      "Group 2d, step 7: 0.0\n",
      "Group 2d, step 8: 0.0\n",
      "Group 2d, step 9: 0.0\n",
      "Group 2e, step 1: 0.9901960784313726\n",
      "Group 2e, step 2: 0.0392156862745098\n",
      "Group 2e, step 3: 0.0392156862745098\n",
      "Group 2e, step 4: 0.0392156862745098\n",
      "Group 2e, step 5: 0.0392156862745098\n",
      "Group 2e, step 6: 0.0\n",
      "Group 2e, step 7: 0.0\n",
      "Group 2e, step 8: 0.0\n",
      "Group 2e, step 9: 0.0\n",
      "Group 2f, step 1: 1.0\n",
      "Group 2f, step 2: 0.9901960784313726\n",
      "Group 2f, step 3: 0.4215686274509804\n",
      "Group 2f, step 4: 0.4215686274509804\n",
      "Group 2f, step 5: 0.0392156862745098\n",
      "Group 2f, step 6: 0.0392156862745098\n",
      "Group 2f, step 7: 0.0\n",
      "Group 2f, step 8: 0.0\n",
      "Group 2f, step 9: 0.0\n",
      "Group 3a, step 1: 0.7009803921568627\n",
      "Group 3a, step 2: 0.0392156862745098\n",
      "Group 3a, step 3: 0.0392156862745098\n",
      "Group 3a, step 4: 0.0\n",
      "Group 3a, step 5: 0.0\n",
      "Group 3a, step 6: 0.0\n",
      "Group 3a, step 7: 0.0\n",
      "Group 3a, step 8: 0.0\n",
      "Group 3a, step 9: 0.0\n",
      "Group 3b, step 1: 0.9901960784313726\n",
      "Group 3b, step 2: 0.9803921568627451\n",
      "Group 3b, step 3: 0.6617647058823529\n",
      "Group 3b, step 4: 0.0\n",
      "Group 3b, step 5: 0.6617647058823529\n",
      "Group 3b, step 6: 0.0\n",
      "Group 3b, step 7: 0.0\n",
      "Group 3b, step 8: 0.0\n",
      "Group 3b, step 9: 0.0\n",
      "Group 3c, step 1: 0.9901960784313726\n",
      "Group 3c, step 2: 0.9901960784313726\n",
      "Group 3c, step 3: 0.0392156862745098\n",
      "Group 3c, step 4: 0.0\n",
      "Group 3c, step 5: 0.0\n",
      "Group 3c, step 6: 0.0\n",
      "Group 3c, step 7: 0.0\n",
      "Group 3c, step 8: 0.0\n",
      "Group 3c, step 9: 0.0\n",
      "Group 3d, step 1: 1.0\n",
      "Group 3d, step 2: 0.9901960784313726\n",
      "Group 3d, step 3: 0.0\n",
      "Group 3d, step 4: 0.0\n",
      "Group 3d, step 5: 0.0\n",
      "Group 3d, step 6: 0.0\n",
      "Group 3d, step 7: 0.0\n",
      "Group 3d, step 8: 0.0\n",
      "Group 3d, step 9: 0.0\n",
      "Group 4a, step 1: 0.9901960784313726\n",
      "Group 4a, step 2: 0.696078431372549\n",
      "Group 4a, step 3: 0.0\n",
      "Group 4a, step 4: 0.0\n",
      "Group 4a, step 5: 0.0\n",
      "Group 4a, step 6: 0.0\n",
      "Group 4a, step 7: 0.0\n",
      "Group 4a, step 8: 0.0\n",
      "Group 4a, step 9: 0.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1a</th>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.990196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1b</th>\n",
       "      <td>0.995098</td>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1c</th>\n",
       "      <td>0.995098</td>\n",
       "      <td>0.985294</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.039216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1d</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2a</th>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2b</th>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.004902</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2c</th>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.950980</td>\n",
       "      <td>0.661765</td>\n",
       "      <td>0.950980</td>\n",
       "      <td>0.950980</td>\n",
       "      <td>0.661765</td>\n",
       "      <td>0.661765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2d</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.656863</td>\n",
       "      <td>0.656863</td>\n",
       "      <td>0.661765</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2e</th>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2f</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.421569</td>\n",
       "      <td>0.421569</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3a</th>\n",
       "      <td>0.700980</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3b</th>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.980392</td>\n",
       "      <td>0.661765</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.661765</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3c</th>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3d</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4a</th>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.696078</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2         3         4         5         6  \\\n",
       "1a  0.990196  0.990196  0.990196  0.990196  0.990196  0.990196  0.990196   \n",
       "1b  0.995098  0.990196  0.039216  0.039216  0.039216  0.000000  0.000000   \n",
       "1c  0.995098  0.985294  0.039216  0.039216  0.039216  0.039216  0.039216   \n",
       "1d  1.000000  1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   \n",
       "2a  0.990196  0.990196  0.039216  0.000000  0.000000  0.000000  0.000000   \n",
       "2b  0.990196  0.990196  0.039216  0.039216  0.039216  0.039216  0.004902   \n",
       "2c  0.990196  0.990196  0.990196  0.950980  0.661765  0.950980  0.950980   \n",
       "2d  1.000000  1.000000  0.656863  0.656863  0.661765  0.000000  0.000000   \n",
       "2e  0.990196  0.039216  0.039216  0.039216  0.039216  0.000000  0.000000   \n",
       "2f  1.000000  0.990196  0.421569  0.421569  0.039216  0.039216  0.000000   \n",
       "3a  0.700980  0.039216  0.039216  0.000000  0.000000  0.000000  0.000000   \n",
       "3b  0.990196  0.980392  0.661765  0.000000  0.661765  0.000000  0.000000   \n",
       "3c  0.990196  0.990196  0.039216  0.000000  0.000000  0.000000  0.000000   \n",
       "3d  1.000000  0.990196  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "4a  0.990196  0.696078  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "\n",
       "           7         8  \n",
       "1a  0.990196  0.990196  \n",
       "1b  0.000000  0.000000  \n",
       "1c  0.039216  0.039216  \n",
       "1d  1.000000  1.000000  \n",
       "2a  0.000000  0.000000  \n",
       "2b  0.000000  0.000000  \n",
       "2c  0.661765  0.661765  \n",
       "2d  0.000000  0.000000  \n",
       "2e  0.000000  0.000000  \n",
       "2f  0.000000  0.000000  \n",
       "3a  0.000000  0.000000  \n",
       "3b  0.000000  0.000000  \n",
       "3c  0.000000  0.000000  \n",
       "3d  0.000000  0.000000  \n",
       "4a  0.000000  0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.999437</td>\n",
       "      <td>0.999437</td>\n",
       "      <td>0.999437</td>\n",
       "      <td>5333.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.988722</td>\n",
       "      <td>0.988722</td>\n",
       "      <td>0.988722</td>\n",
       "      <td>266.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.998928</td>\n",
       "      <td>0.998928</td>\n",
       "      <td>0.998928</td>\n",
       "      <td>0.998928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>macro avg</th>\n",
       "      <td>0.994080</td>\n",
       "      <td>0.994080</td>\n",
       "      <td>0.994080</td>\n",
       "      <td>5599.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weighted avg</th>\n",
       "      <td>0.998928</td>\n",
       "      <td>0.998928</td>\n",
       "      <td>0.998928</td>\n",
       "      <td>5599.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              precision    recall  f1-score      support\n",
       "0              0.999437  0.999437  0.999437  5333.000000\n",
       "1              0.988722  0.988722  0.988722   266.000000\n",
       "accuracy       0.998928  0.998928  0.998928     0.998928\n",
       "macro avg      0.994080  0.994080  0.994080  5599.000000\n",
       "weighted avg   0.998928  0.998928  0.998928  5599.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2c14180975f94d85ba8a5f1e67b01653",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/135 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Group 1a, step 1: 0.9849624060150376\n",
      "Group 1a, step 2: 0.9849624060150376\n",
      "Group 1a, step 3: 0.9849624060150376\n",
      "Group 1a, step 4: 0.9849624060150376\n",
      "Group 1a, step 5: 0.9849624060150376\n",
      "Group 1a, step 6: 0.9849624060150376\n",
      "Group 1a, step 7: 0.981203007518797\n",
      "Group 1a, step 8: 0.9699248120300752\n",
      "Group 1a, step 9: 0.9699248120300752\n",
      "Group 1b, step 1: 0.9887218045112782\n",
      "Group 1b, step 2: 0.9887218045112782\n",
      "Group 1b, step 3: 0.9887218045112782\n",
      "Group 1b, step 4: 0.9887218045112782\n",
      "Group 1b, step 5: 0.981203007518797\n",
      "Group 1b, step 6: 0.9699248120300752\n",
      "Group 1b, step 7: 0.8721804511278195\n",
      "Group 1b, step 8: 0.0\n",
      "Group 1b, step 9: 0.0\n",
      "Group 1c, step 1: 0.9887218045112782\n",
      "Group 1c, step 2: 0.9887218045112782\n",
      "Group 1c, step 3: 0.9849624060150376\n",
      "Group 1c, step 4: 0.9849624060150376\n",
      "Group 1c, step 5: 0.9736842105263158\n",
      "Group 1c, step 6: 0.9736842105263158\n",
      "Group 1c, step 7: 0.9699248120300752\n",
      "Group 1c, step 8: 0.9661654135338346\n",
      "Group 1c, step 9: 0.9661654135338346\n",
      "Group 1d, step 1: 0.981203007518797\n",
      "Group 1d, step 2: 0.9172932330827067\n",
      "Group 1d, step 3: 0.0\n",
      "Group 1d, step 4: 0.0\n",
      "Group 1d, step 5: 0.0\n",
      "Group 1d, step 6: 0.0\n",
      "Group 1d, step 7: 0.0\n",
      "Group 1d, step 8: 0.0\n",
      "Group 1d, step 9: 0.0\n",
      "Group 2a, step 1: 0.9849624060150376\n",
      "Group 2a, step 2: 0.9849624060150376\n",
      "Group 2a, step 3: 0.9849624060150376\n",
      "Group 2a, step 4: 0.9774436090225563\n",
      "Group 2a, step 5: 0.9699248120300752\n",
      "Group 2a, step 6: 0.9699248120300752\n",
      "Group 2a, step 7: 0.0\n",
      "Group 2a, step 8: 0.0\n",
      "Group 2a, step 9: 0.0\n",
      "Group 2b, step 1: 0.9849624060150376\n",
      "Group 2b, step 2: 0.9849624060150376\n",
      "Group 2b, step 3: 0.9849624060150376\n",
      "Group 2b, step 4: 0.9774436090225563\n",
      "Group 2b, step 5: 0.9699248120300752\n",
      "Group 2b, step 6: 0.9586466165413534\n",
      "Group 2b, step 7: 0.0\n",
      "Group 2b, step 8: 0.0\n",
      "Group 2b, step 9: 0.0\n",
      "Group 2c, step 1: 0.981203007518797\n",
      "Group 2c, step 2: 0.924812030075188\n",
      "Group 2c, step 3: 0.0\n",
      "Group 2c, step 4: 0.0\n",
      "Group 2c, step 5: 0.0\n",
      "Group 2c, step 6: 0.0\n",
      "Group 2c, step 7: 0.0\n",
      "Group 2c, step 8: 0.0\n",
      "Group 2c, step 9: 0.0\n",
      "Group 2d, step 1: 0.9360902255639098\n",
      "Group 2d, step 2: 0.8947368421052632\n",
      "Group 2d, step 3: 0.0\n",
      "Group 2d, step 4: 0.0\n",
      "Group 2d, step 5: 0.0\n",
      "Group 2d, step 6: 0.0\n",
      "Group 2d, step 7: 0.0\n",
      "Group 2d, step 8: 0.0\n",
      "Group 2d, step 9: 0.0\n",
      "Group 2e, step 1: 0.9887218045112782\n",
      "Group 2e, step 2: 0.9887218045112782\n",
      "Group 2e, step 3: 0.9887218045112782\n",
      "Group 2e, step 4: 0.9849624060150376\n",
      "Group 2e, step 5: 0.9699248120300752\n",
      "Group 2e, step 6: 0.943609022556391\n",
      "Group 2e, step 7: 0.0\n",
      "Group 2e, step 8: 0.0\n",
      "Group 2e, step 9: 0.0\n",
      "Group 2f, step 1: 0.981203007518797\n",
      "Group 2f, step 2: 0.8947368421052632\n",
      "Group 2f, step 3: 0.0\n",
      "Group 2f, step 4: 0.0\n",
      "Group 2f, step 5: 0.0\n",
      "Group 2f, step 6: 0.0\n",
      "Group 2f, step 7: 0.0\n",
      "Group 2f, step 8: 0.0\n",
      "Group 2f, step 9: 0.0\n",
      "Group 3a, step 1: 0.9849624060150376\n",
      "Group 3a, step 2: 0.9849624060150376\n",
      "Group 3a, step 3: 0.981203007518797\n",
      "Group 3a, step 4: 0.9736842105263158\n",
      "Group 3a, step 5: 0.9699248120300752\n",
      "Group 3a, step 6: 0.7969924812030075\n",
      "Group 3a, step 7: 0.0\n",
      "Group 3a, step 8: 0.0\n",
      "Group 3a, step 9: 0.0\n",
      "Group 3b, step 1: 0.9624060150375939\n",
      "Group 3b, step 2: 0.8984962406015038\n",
      "Group 3b, step 3: 0.0\n",
      "Group 3b, step 4: 0.0\n",
      "Group 3b, step 5: 0.0\n",
      "Group 3b, step 6: 0.0\n",
      "Group 3b, step 7: 0.0\n",
      "Group 3b, step 8: 0.0\n",
      "Group 3b, step 9: 0.0\n",
      "Group 3c, step 1: 0.981203007518797\n",
      "Group 3c, step 2: 0.9172932330827067\n",
      "Group 3c, step 3: 0.0\n",
      "Group 3c, step 4: 0.0\n",
      "Group 3c, step 5: 0.0\n",
      "Group 3c, step 6: 0.0\n",
      "Group 3c, step 7: 0.0\n",
      "Group 3c, step 8: 0.0\n",
      "Group 3c, step 9: 0.0\n",
      "Group 3d, step 1: 0.981203007518797\n",
      "Group 3d, step 2: 0.8571428571428571\n",
      "Group 3d, step 3: 0.0\n",
      "Group 3d, step 4: 0.0\n",
      "Group 3d, step 5: 0.0\n",
      "Group 3d, step 6: 0.0\n",
      "Group 3d, step 7: 0.0\n",
      "Group 3d, step 8: 0.0\n",
      "Group 3d, step 9: 0.0\n",
      "Group 4a, step 1: 0.981203007518797\n",
      "Group 4a, step 2: 0.8646616541353384\n",
      "Group 4a, step 3: 0.0\n",
      "Group 4a, step 4: 0.0\n",
      "Group 4a, step 5: 0.0\n",
      "Group 4a, step 6: 0.0\n",
      "Group 4a, step 7: 0.0\n",
      "Group 4a, step 8: 0.0\n",
      "Group 4a, step 9: 0.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1a</th>\n",
       "      <td>0.984962</td>\n",
       "      <td>0.984962</td>\n",
       "      <td>0.984962</td>\n",
       "      <td>0.984962</td>\n",
       "      <td>0.984962</td>\n",
       "      <td>0.984962</td>\n",
       "      <td>0.981203</td>\n",
       "      <td>0.969925</td>\n",
       "      <td>0.969925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1b</th>\n",
       "      <td>0.988722</td>\n",
       "      <td>0.988722</td>\n",
       "      <td>0.988722</td>\n",
       "      <td>0.988722</td>\n",
       "      <td>0.981203</td>\n",
       "      <td>0.969925</td>\n",
       "      <td>0.872180</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1c</th>\n",
       "      <td>0.988722</td>\n",
       "      <td>0.988722</td>\n",
       "      <td>0.984962</td>\n",
       "      <td>0.984962</td>\n",
       "      <td>0.973684</td>\n",
       "      <td>0.973684</td>\n",
       "      <td>0.969925</td>\n",
       "      <td>0.966165</td>\n",
       "      <td>0.966165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1d</th>\n",
       "      <td>0.981203</td>\n",
       "      <td>0.917293</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2a</th>\n",
       "      <td>0.984962</td>\n",
       "      <td>0.984962</td>\n",
       "      <td>0.984962</td>\n",
       "      <td>0.977444</td>\n",
       "      <td>0.969925</td>\n",
       "      <td>0.969925</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2b</th>\n",
       "      <td>0.984962</td>\n",
       "      <td>0.984962</td>\n",
       "      <td>0.984962</td>\n",
       "      <td>0.977444</td>\n",
       "      <td>0.969925</td>\n",
       "      <td>0.958647</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2c</th>\n",
       "      <td>0.981203</td>\n",
       "      <td>0.924812</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2d</th>\n",
       "      <td>0.936090</td>\n",
       "      <td>0.894737</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2e</th>\n",
       "      <td>0.988722</td>\n",
       "      <td>0.988722</td>\n",
       "      <td>0.988722</td>\n",
       "      <td>0.984962</td>\n",
       "      <td>0.969925</td>\n",
       "      <td>0.943609</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2f</th>\n",
       "      <td>0.981203</td>\n",
       "      <td>0.894737</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3a</th>\n",
       "      <td>0.984962</td>\n",
       "      <td>0.984962</td>\n",
       "      <td>0.981203</td>\n",
       "      <td>0.973684</td>\n",
       "      <td>0.969925</td>\n",
       "      <td>0.796992</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3b</th>\n",
       "      <td>0.962406</td>\n",
       "      <td>0.898496</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3c</th>\n",
       "      <td>0.981203</td>\n",
       "      <td>0.917293</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3d</th>\n",
       "      <td>0.981203</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4a</th>\n",
       "      <td>0.981203</td>\n",
       "      <td>0.864662</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2         3         4         5         6  \\\n",
       "1a  0.984962  0.984962  0.984962  0.984962  0.984962  0.984962  0.981203   \n",
       "1b  0.988722  0.988722  0.988722  0.988722  0.981203  0.969925  0.872180   \n",
       "1c  0.988722  0.988722  0.984962  0.984962  0.973684  0.973684  0.969925   \n",
       "1d  0.981203  0.917293  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "2a  0.984962  0.984962  0.984962  0.977444  0.969925  0.969925  0.000000   \n",
       "2b  0.984962  0.984962  0.984962  0.977444  0.969925  0.958647  0.000000   \n",
       "2c  0.981203  0.924812  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "2d  0.936090  0.894737  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "2e  0.988722  0.988722  0.988722  0.984962  0.969925  0.943609  0.000000   \n",
       "2f  0.981203  0.894737  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "3a  0.984962  0.984962  0.981203  0.973684  0.969925  0.796992  0.000000   \n",
       "3b  0.962406  0.898496  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "3c  0.981203  0.917293  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "3d  0.981203  0.857143  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "4a  0.981203  0.864662  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "\n",
       "           7         8  \n",
       "1a  0.969925  0.969925  \n",
       "1b  0.000000  0.000000  \n",
       "1c  0.966165  0.966165  \n",
       "1d  0.000000  0.000000  \n",
       "2a  0.000000  0.000000  \n",
       "2b  0.000000  0.000000  \n",
       "2c  0.000000  0.000000  \n",
       "2d  0.000000  0.000000  \n",
       "2e  0.000000  0.000000  \n",
       "2f  0.000000  0.000000  \n",
       "3a  0.000000  0.000000  \n",
       "3b  0.000000  0.000000  \n",
       "3c  0.000000  0.000000  \n",
       "3d  0.000000  0.000000  \n",
       "4a  0.000000  0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if not os.path.exists('results/CTU/scores/rf/'):\n",
    "    os.mkdir('results/CTU/scores/rf/')\n",
    "\n",
    "for attack in ['neris', 'rbot', 'virut', 'menti', 'murlo']:\n",
    "\n",
    "    if not os.path.exists(f'results/CTU/scores/rf/{attack}'):\n",
    "        os.mkdir(f'results/CTU/scores/rf/{attack}')\n",
    "    \n",
    "    train_data = pd.read_csv(f'preprocessed_data/CTU/{attack}_train.csv')\n",
    "    test_data = pd.read_csv(f'preprocessed_data/CTU/{attack}_test.csv')\n",
    "    train_data.drop(columns=['Src IP', 'Dst IP'], inplace=True)\n",
    "    test_data.drop(columns=['Src IP', 'Dst IP'], inplace=True)\n",
    "    \n",
    "    model = RandomForestClassifier(n_estimators=100, n_jobs=-1)\n",
    "    model.fit(train_data.drop(columns=['Label']), train_data.Label)\n",
    "    baseline_pred = model.predict(test_data.drop(columns=['Label']))\n",
    "    report = classification_report(test_data.Label, baseline_pred, digits=3, output_dict=True)\n",
    "    report = pd.DataFrame(report).transpose()\n",
    "    display(report)\n",
    "    \n",
    "    report.to_csv(f'results/CTU/scores/rf/{attack}/baseline.csv')\n",
    "    \n",
    "    recall, precision, f1 = adversarial_feature_attack(test_data, model)\n",
    "    display(recall)\n",
    "    \n",
    "    recall.to_csv(f'results/CTU/scores/rf/{attack}/feature_attack.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.999796</td>\n",
       "      <td>0.999946</td>\n",
       "      <td>0.999871</td>\n",
       "      <td>73650.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.998910</td>\n",
       "      <td>0.995926</td>\n",
       "      <td>0.997416</td>\n",
       "      <td>3682.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.999754</td>\n",
       "      <td>0.999754</td>\n",
       "      <td>0.999754</td>\n",
       "      <td>0.999754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>macro avg</th>\n",
       "      <td>0.999353</td>\n",
       "      <td>0.997936</td>\n",
       "      <td>0.998644</td>\n",
       "      <td>77332.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weighted avg</th>\n",
       "      <td>0.999754</td>\n",
       "      <td>0.999754</td>\n",
       "      <td>0.999754</td>\n",
       "      <td>77332.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              precision    recall  f1-score       support\n",
       "0              0.999796  0.999946  0.999871  73650.000000\n",
       "1              0.998910  0.995926  0.997416   3682.000000\n",
       "accuracy       0.999754  0.999754  0.999754      0.999754\n",
       "macro avg      0.999353  0.997936  0.998644  77332.000000\n",
       "weighted avg   0.999754  0.999754  0.999754  77332.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c346c50b22a47d9928d0175cc50f4e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/135 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Group 1a, step 1: 0.9961977186311787\n",
      "Group 1a, step 2: 0.9959261271048343\n",
      "Group 1a, step 3: 0.9956545355784899\n",
      "Group 1a, step 4: 0.9959261271048343\n",
      "Group 1a, step 5: 0.996469310157523\n",
      "Group 1a, step 6: 0.996469310157523\n",
      "Group 1a, step 7: 0.996469310157523\n",
      "Group 1a, step 8: 0.9961977186311787\n",
      "Group 1a, step 9: 0.9940249864204237\n",
      "Group 1b, step 1: 0.9956545355784899\n",
      "Group 1b, step 2: 0.9956545355784899\n",
      "Group 1b, step 3: 0.9953829440521456\n",
      "Group 1b, step 4: 0.9953829440521456\n",
      "Group 1b, step 5: 0.9951113525258012\n",
      "Group 1b, step 6: 0.9937533948940793\n",
      "Group 1b, step 7: 0.9923954372623575\n",
      "Group 1b, step 8: 0.989951113525258\n",
      "Group 1b, step 9: 0.9896795219989136\n",
      "Group 1c, step 1: 0.9956545355784899\n",
      "Group 1c, step 2: 0.9956545355784899\n",
      "Group 1c, step 3: 0.9956545355784899\n",
      "Group 1c, step 4: 0.9956545355784899\n",
      "Group 1c, step 5: 0.9951113525258012\n",
      "Group 1c, step 6: 0.9951113525258012\n",
      "Group 1c, step 7: 0.9942965779467681\n",
      "Group 1c, step 8: 0.9926670287887018\n",
      "Group 1c, step 9: 0.9915806626833242\n",
      "Group 1d, step 1: 0.9959261271048343\n",
      "Group 1d, step 2: 0.9959261271048343\n",
      "Group 1d, step 3: 0.9956545355784899\n",
      "Group 1d, step 4: 0.9953829440521456\n",
      "Group 1d, step 5: 0.9953829440521456\n",
      "Group 1d, step 6: 0.9953829440521456\n",
      "Group 1d, step 7: 0.9953829440521456\n",
      "Group 1d, step 8: 0.9953829440521456\n",
      "Group 1d, step 9: 0.9953829440521456\n",
      "Group 2a, step 1: 0.9959261271048343\n",
      "Group 2a, step 2: 0.9956545355784899\n",
      "Group 2a, step 3: 0.9951113525258012\n",
      "Group 2a, step 4: 0.9953829440521456\n",
      "Group 2a, step 5: 0.9915806626833242\n",
      "Group 2a, step 6: 0.9894079304725693\n",
      "Group 2a, step 7: 0.9885931558935361\n",
      "Group 2a, step 8: 0.9888647474198805\n",
      "Group 2a, step 9: 0.9600760456273765\n",
      "Group 2b, step 1: 0.9961977186311787\n",
      "Group 2b, step 2: 0.9959261271048343\n",
      "Group 2b, step 3: 0.9956545355784899\n",
      "Group 2b, step 4: 0.9956545355784899\n",
      "Group 2b, step 5: 0.9929386203150462\n",
      "Group 2b, step 6: 0.9896795219989136\n",
      "Group 2b, step 7: 0.9896795219989136\n",
      "Group 2b, step 8: 0.9888647474198805\n",
      "Group 2b, step 9: 0.9891363389462249\n",
      "Group 2c, step 1: 0.9961977186311787\n",
      "Group 2c, step 2: 0.9959261271048343\n",
      "Group 2c, step 3: 0.9959261271048343\n",
      "Group 2c, step 4: 0.9959261271048343\n",
      "Group 2c, step 5: 0.9953829440521456\n",
      "Group 2c, step 6: 0.9948397609994568\n",
      "Group 2c, step 7: 0.9940249864204237\n",
      "Group 2c, step 8: 0.9891363389462249\n",
      "Group 2c, step 9: 0.9869636067354699\n",
      "Group 2d, step 1: 0.9953829440521456\n",
      "Group 2d, step 2: 0.9953829440521456\n",
      "Group 2d, step 3: 0.9951113525258012\n",
      "Group 2d, step 4: 0.9948397609994568\n",
      "Group 2d, step 5: 0.9918522542096686\n",
      "Group 2d, step 6: 0.9894079304725693\n",
      "Group 2d, step 7: 0.9880499728408474\n",
      "Group 2d, step 8: 0.9861488321564367\n",
      "Group 2d, step 9: 0.9505703422053232\n",
      "Group 2e, step 1: 0.9953829440521456\n",
      "Group 2e, step 2: 0.9953829440521456\n",
      "Group 2e, step 3: 0.9953829440521456\n",
      "Group 2e, step 4: 0.9953829440521456\n",
      "Group 2e, step 5: 0.9918522542096686\n",
      "Group 2e, step 6: 0.9894079304725693\n",
      "Group 2e, step 7: 0.9894079304725693\n",
      "Group 2e, step 8: 0.987778381314503\n",
      "Group 2e, step 9: 0.9516567083107007\n",
      "Group 2f, step 1: 0.9956545355784899\n",
      "Group 2f, step 2: 0.9956545355784899\n",
      "Group 2f, step 3: 0.9953829440521456\n",
      "Group 2f, step 4: 0.9948397609994568\n",
      "Group 2f, step 5: 0.9942965779467681\n",
      "Group 2f, step 6: 0.9921238457360131\n",
      "Group 2f, step 7: 0.9896795219989136\n",
      "Group 2f, step 8: 0.9891363389462249\n",
      "Group 2f, step 9: 0.9875067897881586\n",
      "Group 3a, step 1: 0.9959261271048343\n",
      "Group 3a, step 2: 0.9956545355784899\n",
      "Group 3a, step 3: 0.9951113525258012\n",
      "Group 3a, step 4: 0.9953829440521456\n",
      "Group 3a, step 5: 0.9904942965779467\n",
      "Group 3a, step 6: 0.9872351982618143\n",
      "Group 3a, step 7: 0.9872351982618143\n",
      "Group 3a, step 8: 0.9883215643671918\n",
      "Group 3a, step 9: 0.9494839760999457\n",
      "Group 3b, step 1: 0.9956545355784899\n",
      "Group 3b, step 2: 0.9953829440521456\n",
      "Group 3b, step 3: 0.9948397609994568\n",
      "Group 3b, step 4: 0.9937533948940793\n",
      "Group 3b, step 5: 0.9894079304725693\n",
      "Group 3b, step 6: 0.9725692558392178\n",
      "Group 3b, step 7: 0.9644215100488864\n",
      "Group 3b, step 8: 0.9527430744160782\n",
      "Group 3b, step 9: 0.9473112438891906\n",
      "Group 3c, step 1: 0.9961977186311787\n",
      "Group 3c, step 2: 0.9959261271048343\n",
      "Group 3c, step 3: 0.9953829440521456\n",
      "Group 3c, step 4: 0.9942965779467681\n",
      "Group 3c, step 5: 0.9896795219989136\n",
      "Group 3c, step 6: 0.9853340575774036\n",
      "Group 3c, step 7: 0.9850624660510592\n",
      "Group 3c, step 8: 0.9880499728408474\n",
      "Group 3c, step 9: 0.9850624660510592\n",
      "Group 3d, step 1: 0.9953829440521456\n",
      "Group 3d, step 2: 0.9953829440521456\n",
      "Group 3d, step 3: 0.9951113525258012\n",
      "Group 3d, step 4: 0.9951113525258012\n",
      "Group 3d, step 5: 0.9894079304725693\n",
      "Group 3d, step 6: 0.9858772406300923\n",
      "Group 3d, step 7: 0.9845192829983704\n",
      "Group 3d, step 8: 0.9619771863117871\n",
      "Group 3d, step 9: 0.94541010320478\n",
      "Group 4a, step 1: 0.9959261271048343\n",
      "Group 4a, step 2: 0.9956545355784899\n",
      "Group 4a, step 3: 0.9948397609994568\n",
      "Group 4a, step 4: 0.993481803367735\n",
      "Group 4a, step 5: 0.9869636067354699\n",
      "Group 4a, step 6: 0.9728408473655622\n",
      "Group 4a, step 7: 0.9845192829983704\n",
      "Group 4a, step 8: 0.9508419337316676\n",
      "Group 4a, step 9: 0.9464964693101575\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1a</th>\n",
       "      <td>0.996198</td>\n",
       "      <td>0.995926</td>\n",
       "      <td>0.995655</td>\n",
       "      <td>0.995926</td>\n",
       "      <td>0.996469</td>\n",
       "      <td>0.996469</td>\n",
       "      <td>0.996469</td>\n",
       "      <td>0.996198</td>\n",
       "      <td>0.994025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1b</th>\n",
       "      <td>0.995655</td>\n",
       "      <td>0.995655</td>\n",
       "      <td>0.995383</td>\n",
       "      <td>0.995383</td>\n",
       "      <td>0.995111</td>\n",
       "      <td>0.993753</td>\n",
       "      <td>0.992395</td>\n",
       "      <td>0.989951</td>\n",
       "      <td>0.989680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1c</th>\n",
       "      <td>0.995655</td>\n",
       "      <td>0.995655</td>\n",
       "      <td>0.995655</td>\n",
       "      <td>0.995655</td>\n",
       "      <td>0.995111</td>\n",
       "      <td>0.995111</td>\n",
       "      <td>0.994297</td>\n",
       "      <td>0.992667</td>\n",
       "      <td>0.991581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1d</th>\n",
       "      <td>0.995926</td>\n",
       "      <td>0.995926</td>\n",
       "      <td>0.995655</td>\n",
       "      <td>0.995383</td>\n",
       "      <td>0.995383</td>\n",
       "      <td>0.995383</td>\n",
       "      <td>0.995383</td>\n",
       "      <td>0.995383</td>\n",
       "      <td>0.995383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2a</th>\n",
       "      <td>0.995926</td>\n",
       "      <td>0.995655</td>\n",
       "      <td>0.995111</td>\n",
       "      <td>0.995383</td>\n",
       "      <td>0.991581</td>\n",
       "      <td>0.989408</td>\n",
       "      <td>0.988593</td>\n",
       "      <td>0.988865</td>\n",
       "      <td>0.960076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2b</th>\n",
       "      <td>0.996198</td>\n",
       "      <td>0.995926</td>\n",
       "      <td>0.995655</td>\n",
       "      <td>0.995655</td>\n",
       "      <td>0.992939</td>\n",
       "      <td>0.989680</td>\n",
       "      <td>0.989680</td>\n",
       "      <td>0.988865</td>\n",
       "      <td>0.989136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2c</th>\n",
       "      <td>0.996198</td>\n",
       "      <td>0.995926</td>\n",
       "      <td>0.995926</td>\n",
       "      <td>0.995926</td>\n",
       "      <td>0.995383</td>\n",
       "      <td>0.994840</td>\n",
       "      <td>0.994025</td>\n",
       "      <td>0.989136</td>\n",
       "      <td>0.986964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2d</th>\n",
       "      <td>0.995383</td>\n",
       "      <td>0.995383</td>\n",
       "      <td>0.995111</td>\n",
       "      <td>0.994840</td>\n",
       "      <td>0.991852</td>\n",
       "      <td>0.989408</td>\n",
       "      <td>0.988050</td>\n",
       "      <td>0.986149</td>\n",
       "      <td>0.950570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2e</th>\n",
       "      <td>0.995383</td>\n",
       "      <td>0.995383</td>\n",
       "      <td>0.995383</td>\n",
       "      <td>0.995383</td>\n",
       "      <td>0.991852</td>\n",
       "      <td>0.989408</td>\n",
       "      <td>0.989408</td>\n",
       "      <td>0.987778</td>\n",
       "      <td>0.951657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2f</th>\n",
       "      <td>0.995655</td>\n",
       "      <td>0.995655</td>\n",
       "      <td>0.995383</td>\n",
       "      <td>0.994840</td>\n",
       "      <td>0.994297</td>\n",
       "      <td>0.992124</td>\n",
       "      <td>0.989680</td>\n",
       "      <td>0.989136</td>\n",
       "      <td>0.987507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3a</th>\n",
       "      <td>0.995926</td>\n",
       "      <td>0.995655</td>\n",
       "      <td>0.995111</td>\n",
       "      <td>0.995383</td>\n",
       "      <td>0.990494</td>\n",
       "      <td>0.987235</td>\n",
       "      <td>0.987235</td>\n",
       "      <td>0.988322</td>\n",
       "      <td>0.949484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3b</th>\n",
       "      <td>0.995655</td>\n",
       "      <td>0.995383</td>\n",
       "      <td>0.994840</td>\n",
       "      <td>0.993753</td>\n",
       "      <td>0.989408</td>\n",
       "      <td>0.972569</td>\n",
       "      <td>0.964422</td>\n",
       "      <td>0.952743</td>\n",
       "      <td>0.947311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3c</th>\n",
       "      <td>0.996198</td>\n",
       "      <td>0.995926</td>\n",
       "      <td>0.995383</td>\n",
       "      <td>0.994297</td>\n",
       "      <td>0.989680</td>\n",
       "      <td>0.985334</td>\n",
       "      <td>0.985062</td>\n",
       "      <td>0.988050</td>\n",
       "      <td>0.985062</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3d</th>\n",
       "      <td>0.995383</td>\n",
       "      <td>0.995383</td>\n",
       "      <td>0.995111</td>\n",
       "      <td>0.995111</td>\n",
       "      <td>0.989408</td>\n",
       "      <td>0.985877</td>\n",
       "      <td>0.984519</td>\n",
       "      <td>0.961977</td>\n",
       "      <td>0.945410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4a</th>\n",
       "      <td>0.995926</td>\n",
       "      <td>0.995655</td>\n",
       "      <td>0.994840</td>\n",
       "      <td>0.993482</td>\n",
       "      <td>0.986964</td>\n",
       "      <td>0.972841</td>\n",
       "      <td>0.984519</td>\n",
       "      <td>0.950842</td>\n",
       "      <td>0.946496</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2         3         4         5         6  \\\n",
       "1a  0.996198  0.995926  0.995655  0.995926  0.996469  0.996469  0.996469   \n",
       "1b  0.995655  0.995655  0.995383  0.995383  0.995111  0.993753  0.992395   \n",
       "1c  0.995655  0.995655  0.995655  0.995655  0.995111  0.995111  0.994297   \n",
       "1d  0.995926  0.995926  0.995655  0.995383  0.995383  0.995383  0.995383   \n",
       "2a  0.995926  0.995655  0.995111  0.995383  0.991581  0.989408  0.988593   \n",
       "2b  0.996198  0.995926  0.995655  0.995655  0.992939  0.989680  0.989680   \n",
       "2c  0.996198  0.995926  0.995926  0.995926  0.995383  0.994840  0.994025   \n",
       "2d  0.995383  0.995383  0.995111  0.994840  0.991852  0.989408  0.988050   \n",
       "2e  0.995383  0.995383  0.995383  0.995383  0.991852  0.989408  0.989408   \n",
       "2f  0.995655  0.995655  0.995383  0.994840  0.994297  0.992124  0.989680   \n",
       "3a  0.995926  0.995655  0.995111  0.995383  0.990494  0.987235  0.987235   \n",
       "3b  0.995655  0.995383  0.994840  0.993753  0.989408  0.972569  0.964422   \n",
       "3c  0.996198  0.995926  0.995383  0.994297  0.989680  0.985334  0.985062   \n",
       "3d  0.995383  0.995383  0.995111  0.995111  0.989408  0.985877  0.984519   \n",
       "4a  0.995926  0.995655  0.994840  0.993482  0.986964  0.972841  0.984519   \n",
       "\n",
       "           7         8  \n",
       "1a  0.996198  0.994025  \n",
       "1b  0.989951  0.989680  \n",
       "1c  0.992667  0.991581  \n",
       "1d  0.995383  0.995383  \n",
       "2a  0.988865  0.960076  \n",
       "2b  0.988865  0.989136  \n",
       "2c  0.989136  0.986964  \n",
       "2d  0.986149  0.950570  \n",
       "2e  0.987778  0.951657  \n",
       "2f  0.989136  0.987507  \n",
       "3a  0.988322  0.949484  \n",
       "3b  0.952743  0.947311  \n",
       "3c  0.988050  0.985062  \n",
       "3d  0.961977  0.945410  \n",
       "4a  0.950842  0.946496  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.999620</td>\n",
       "      <td>0.999740</td>\n",
       "      <td>0.999680</td>\n",
       "      <td>99950.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.994785</td>\n",
       "      <td>0.992397</td>\n",
       "      <td>0.993590</td>\n",
       "      <td>4998.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.999390</td>\n",
       "      <td>0.999390</td>\n",
       "      <td>0.999390</td>\n",
       "      <td>0.99939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>macro avg</th>\n",
       "      <td>0.997203</td>\n",
       "      <td>0.996068</td>\n",
       "      <td>0.996635</td>\n",
       "      <td>104948.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weighted avg</th>\n",
       "      <td>0.999390</td>\n",
       "      <td>0.999390</td>\n",
       "      <td>0.999390</td>\n",
       "      <td>104948.00000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              precision    recall  f1-score       support\n",
       "0              0.999620  0.999740  0.999680   99950.00000\n",
       "1              0.994785  0.992397  0.993590    4998.00000\n",
       "accuracy       0.999390  0.999390  0.999390       0.99939\n",
       "macro avg      0.997203  0.996068  0.996635  104948.00000\n",
       "weighted avg   0.999390  0.999390  0.999390  104948.00000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "06c764f00658491db1c6727f25b59714",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/135 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Group 1a, step 1: 0.9891956782713085\n",
      "Group 1a, step 2: 0.9891956782713085\n",
      "Group 1a, step 3: 0.9891956782713085\n",
      "Group 1a, step 4: 0.9891956782713085\n",
      "Group 1a, step 5: 0.9891956782713085\n",
      "Group 1a, step 6: 0.9891956782713085\n",
      "Group 1a, step 7: 0.9891956782713085\n",
      "Group 1a, step 8: 0.9891956782713085\n",
      "Group 1a, step 9: 0.9891956782713085\n",
      "Group 1b, step 1: 0.9919967987194878\n",
      "Group 1b, step 2: 0.9919967987194878\n",
      "Group 1b, step 3: 0.9919967987194878\n",
      "Group 1b, step 4: 0.9919967987194878\n",
      "Group 1b, step 5: 0.9919967987194878\n",
      "Group 1b, step 6: 0.9911964785914366\n",
      "Group 1b, step 7: 0.9911964785914366\n",
      "Group 1b, step 8: 0.9911964785914366\n",
      "Group 1b, step 9: 0.9911964785914366\n",
      "Group 1c, step 1: 0.9919967987194878\n",
      "Group 1c, step 2: 0.9919967987194878\n",
      "Group 1c, step 3: 0.9919967987194878\n",
      "Group 1c, step 4: 0.9919967987194878\n",
      "Group 1c, step 5: 0.9919967987194878\n",
      "Group 1c, step 6: 0.9915966386554622\n",
      "Group 1c, step 7: 0.9915966386554622\n",
      "Group 1c, step 8: 0.9911964785914366\n",
      "Group 1c, step 9: 0.9911964785914366\n",
      "Group 1d, step 1: 0.9905962384953981\n",
      "Group 1d, step 2: 0.9905962384953981\n",
      "Group 1d, step 3: 0.9905962384953981\n",
      "Group 1d, step 4: 0.9905962384953981\n",
      "Group 1d, step 5: 0.9905962384953981\n",
      "Group 1d, step 6: 0.9905962384953981\n",
      "Group 1d, step 7: 0.9905962384953981\n",
      "Group 1d, step 8: 0.9905962384953981\n",
      "Group 1d, step 9: 0.9905962384953981\n",
      "Group 2a, step 1: 0.9891956782713085\n",
      "Group 2a, step 2: 0.9891956782713085\n",
      "Group 2a, step 3: 0.9891956782713085\n",
      "Group 2a, step 4: 0.9891956782713085\n",
      "Group 2a, step 5: 0.9891956782713085\n",
      "Group 2a, step 6: 0.9891956782713085\n",
      "Group 2a, step 7: 0.9891956782713085\n",
      "Group 2a, step 8: 0.9889955982392957\n",
      "Group 2a, step 9: 0.9889955982392957\n",
      "Group 2b, step 1: 0.9891956782713085\n",
      "Group 2b, step 2: 0.9891956782713085\n",
      "Group 2b, step 3: 0.9891956782713085\n",
      "Group 2b, step 4: 0.9891956782713085\n",
      "Group 2b, step 5: 0.9891956782713085\n",
      "Group 2b, step 6: 0.9891956782713085\n",
      "Group 2b, step 7: 0.9891956782713085\n",
      "Group 2b, step 8: 0.9889955982392957\n",
      "Group 2b, step 9: 0.9889955982392957\n",
      "Group 2c, step 1: 0.9891956782713085\n",
      "Group 2c, step 2: 0.9891956782713085\n",
      "Group 2c, step 3: 0.9891956782713085\n",
      "Group 2c, step 4: 0.9891956782713085\n",
      "Group 2c, step 5: 0.9891956782713085\n",
      "Group 2c, step 6: 0.9891956782713085\n",
      "Group 2c, step 7: 0.9891956782713085\n",
      "Group 2c, step 8: 0.9891956782713085\n",
      "Group 2c, step 9: 0.9891956782713085\n",
      "Group 2d, step 1: 0.9901960784313726\n",
      "Group 2d, step 2: 0.9901960784313726\n",
      "Group 2d, step 3: 0.9897959183673469\n",
      "Group 2d, step 4: 0.9895958383353342\n",
      "Group 2d, step 5: 0.9895958383353342\n",
      "Group 2d, step 6: 0.9891956782713085\n",
      "Group 2d, step 7: 0.9891956782713085\n",
      "Group 2d, step 8: 0.9891956782713085\n",
      "Group 2d, step 9: 0.9891956782713085\n",
      "Group 2e, step 1: 0.9919967987194878\n",
      "Group 2e, step 2: 0.9919967987194878\n",
      "Group 2e, step 3: 0.991796718687475\n",
      "Group 2e, step 4: 0.991796718687475\n",
      "Group 2e, step 5: 0.9911964785914366\n",
      "Group 2e, step 6: 0.9911964785914366\n",
      "Group 2e, step 7: 0.9909963985594238\n",
      "Group 2e, step 8: 0.9909963985594238\n",
      "Group 2e, step 9: 0.9909963985594238\n",
      "Group 2f, step 1: 0.9903961584633854\n",
      "Group 2f, step 2: 0.9901960784313726\n",
      "Group 2f, step 3: 0.9897959183673469\n",
      "Group 2f, step 4: 0.9895958383353342\n",
      "Group 2f, step 5: 0.9895958383353342\n",
      "Group 2f, step 6: 0.9891956782713085\n",
      "Group 2f, step 7: 0.9891956782713085\n",
      "Group 2f, step 8: 0.9891956782713085\n",
      "Group 2f, step 9: 0.9891956782713085\n",
      "Group 3a, step 1: 0.9891956782713085\n",
      "Group 3a, step 2: 0.9891956782713085\n",
      "Group 3a, step 3: 0.9891956782713085\n",
      "Group 3a, step 4: 0.9891956782713085\n",
      "Group 3a, step 5: 0.9891956782713085\n",
      "Group 3a, step 6: 0.9891956782713085\n",
      "Group 3a, step 7: 0.9889955982392957\n",
      "Group 3a, step 8: 0.9889955982392957\n",
      "Group 3a, step 9: 0.9889955982392957\n",
      "Group 3b, step 1: 0.9891956782713085\n",
      "Group 3b, step 2: 0.48999599839935976\n",
      "Group 3b, step 3: 0.0070028011204481795\n",
      "Group 3b, step 4: 0.0070028011204481795\n",
      "Group 3b, step 5: 0.0070028011204481795\n",
      "Group 3b, step 6: 0.0070028011204481795\n",
      "Group 3b, step 7: 0.006802721088435374\n",
      "Group 3b, step 8: 0.006802721088435374\n",
      "Group 3b, step 9: 0.006802721088435374\n",
      "Group 3c, step 1: 0.9891956782713085\n",
      "Group 3c, step 2: 0.9891956782713085\n",
      "Group 3c, step 3: 0.9891956782713085\n",
      "Group 3c, step 4: 0.48999599839935976\n",
      "Group 3c, step 5: 0.48999599839935976\n",
      "Group 3c, step 6: 0.48999599839935976\n",
      "Group 3c, step 7: 0.48999599839935976\n",
      "Group 3c, step 8: 0.4897959183673469\n",
      "Group 3c, step 9: 0.4897959183673469\n",
      "Group 3d, step 1: 0.9895958383353342\n",
      "Group 3d, step 2: 0.9895958383353342\n",
      "Group 3d, step 3: 0.9891956782713085\n",
      "Group 3d, step 4: 0.9891956782713085\n",
      "Group 3d, step 5: 0.9891956782713085\n",
      "Group 3d, step 6: 0.9299719887955182\n",
      "Group 3d, step 7: 0.9103641456582633\n",
      "Group 3d, step 8: 0.9103641456582633\n",
      "Group 3d, step 9: 0.9103641456582633\n",
      "Group 4a, step 1: 0.48999599839935976\n",
      "Group 4a, step 2: 0.0070028011204481795\n",
      "Group 4a, step 3: 0.0070028011204481795\n",
      "Group 4a, step 4: 0.0070028011204481795\n",
      "Group 4a, step 5: 0.0070028011204481795\n",
      "Group 4a, step 6: 0.0070028011204481795\n",
      "Group 4a, step 7: 0.006802721088435374\n",
      "Group 4a, step 8: 0.006802721088435374\n",
      "Group 4a, step 9: 0.006802721088435374\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1a</th>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1b</th>\n",
       "      <td>0.991997</td>\n",
       "      <td>0.991997</td>\n",
       "      <td>0.991997</td>\n",
       "      <td>0.991997</td>\n",
       "      <td>0.991997</td>\n",
       "      <td>0.991196</td>\n",
       "      <td>0.991196</td>\n",
       "      <td>0.991196</td>\n",
       "      <td>0.991196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1c</th>\n",
       "      <td>0.991997</td>\n",
       "      <td>0.991997</td>\n",
       "      <td>0.991997</td>\n",
       "      <td>0.991997</td>\n",
       "      <td>0.991997</td>\n",
       "      <td>0.991597</td>\n",
       "      <td>0.991597</td>\n",
       "      <td>0.991196</td>\n",
       "      <td>0.991196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1d</th>\n",
       "      <td>0.990596</td>\n",
       "      <td>0.990596</td>\n",
       "      <td>0.990596</td>\n",
       "      <td>0.990596</td>\n",
       "      <td>0.990596</td>\n",
       "      <td>0.990596</td>\n",
       "      <td>0.990596</td>\n",
       "      <td>0.990596</td>\n",
       "      <td>0.990596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2a</th>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.988996</td>\n",
       "      <td>0.988996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2b</th>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.988996</td>\n",
       "      <td>0.988996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2c</th>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2d</th>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.989796</td>\n",
       "      <td>0.989596</td>\n",
       "      <td>0.989596</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2e</th>\n",
       "      <td>0.991997</td>\n",
       "      <td>0.991997</td>\n",
       "      <td>0.991797</td>\n",
       "      <td>0.991797</td>\n",
       "      <td>0.991196</td>\n",
       "      <td>0.991196</td>\n",
       "      <td>0.990996</td>\n",
       "      <td>0.990996</td>\n",
       "      <td>0.990996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2f</th>\n",
       "      <td>0.990396</td>\n",
       "      <td>0.990196</td>\n",
       "      <td>0.989796</td>\n",
       "      <td>0.989596</td>\n",
       "      <td>0.989596</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3a</th>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.988996</td>\n",
       "      <td>0.988996</td>\n",
       "      <td>0.988996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3b</th>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.489996</td>\n",
       "      <td>0.007003</td>\n",
       "      <td>0.007003</td>\n",
       "      <td>0.007003</td>\n",
       "      <td>0.007003</td>\n",
       "      <td>0.006803</td>\n",
       "      <td>0.006803</td>\n",
       "      <td>0.006803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3c</th>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.489996</td>\n",
       "      <td>0.489996</td>\n",
       "      <td>0.489996</td>\n",
       "      <td>0.489996</td>\n",
       "      <td>0.489796</td>\n",
       "      <td>0.489796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3d</th>\n",
       "      <td>0.989596</td>\n",
       "      <td>0.989596</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.989196</td>\n",
       "      <td>0.929972</td>\n",
       "      <td>0.910364</td>\n",
       "      <td>0.910364</td>\n",
       "      <td>0.910364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4a</th>\n",
       "      <td>0.489996</td>\n",
       "      <td>0.007003</td>\n",
       "      <td>0.007003</td>\n",
       "      <td>0.007003</td>\n",
       "      <td>0.007003</td>\n",
       "      <td>0.007003</td>\n",
       "      <td>0.006803</td>\n",
       "      <td>0.006803</td>\n",
       "      <td>0.006803</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2         3         4         5         6  \\\n",
       "1a  0.989196  0.989196  0.989196  0.989196  0.989196  0.989196  0.989196   \n",
       "1b  0.991997  0.991997  0.991997  0.991997  0.991997  0.991196  0.991196   \n",
       "1c  0.991997  0.991997  0.991997  0.991997  0.991997  0.991597  0.991597   \n",
       "1d  0.990596  0.990596  0.990596  0.990596  0.990596  0.990596  0.990596   \n",
       "2a  0.989196  0.989196  0.989196  0.989196  0.989196  0.989196  0.989196   \n",
       "2b  0.989196  0.989196  0.989196  0.989196  0.989196  0.989196  0.989196   \n",
       "2c  0.989196  0.989196  0.989196  0.989196  0.989196  0.989196  0.989196   \n",
       "2d  0.990196  0.990196  0.989796  0.989596  0.989596  0.989196  0.989196   \n",
       "2e  0.991997  0.991997  0.991797  0.991797  0.991196  0.991196  0.990996   \n",
       "2f  0.990396  0.990196  0.989796  0.989596  0.989596  0.989196  0.989196   \n",
       "3a  0.989196  0.989196  0.989196  0.989196  0.989196  0.989196  0.988996   \n",
       "3b  0.989196  0.489996  0.007003  0.007003  0.007003  0.007003  0.006803   \n",
       "3c  0.989196  0.989196  0.989196  0.489996  0.489996  0.489996  0.489996   \n",
       "3d  0.989596  0.989596  0.989196  0.989196  0.989196  0.929972  0.910364   \n",
       "4a  0.489996  0.007003  0.007003  0.007003  0.007003  0.007003  0.006803   \n",
       "\n",
       "           7         8  \n",
       "1a  0.989196  0.989196  \n",
       "1b  0.991196  0.991196  \n",
       "1c  0.991196  0.991196  \n",
       "1d  0.990596  0.990596  \n",
       "2a  0.988996  0.988996  \n",
       "2b  0.988996  0.988996  \n",
       "2c  0.989196  0.989196  \n",
       "2d  0.989196  0.989196  \n",
       "2e  0.990996  0.990996  \n",
       "2f  0.989196  0.989196  \n",
       "3a  0.988996  0.988996  \n",
       "3b  0.006803  0.006803  \n",
       "3c  0.489796  0.489796  \n",
       "3d  0.910364  0.910364  \n",
       "4a  0.006803  0.006803  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.999568</td>\n",
       "      <td>0.999935</td>\n",
       "      <td>0.999752</td>\n",
       "      <td>92600.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.998695</td>\n",
       "      <td>0.991361</td>\n",
       "      <td>0.995014</td>\n",
       "      <td>4630.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.999527</td>\n",
       "      <td>0.999527</td>\n",
       "      <td>0.999527</td>\n",
       "      <td>0.999527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>macro avg</th>\n",
       "      <td>0.999131</td>\n",
       "      <td>0.995648</td>\n",
       "      <td>0.997383</td>\n",
       "      <td>97230.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weighted avg</th>\n",
       "      <td>0.999527</td>\n",
       "      <td>0.999527</td>\n",
       "      <td>0.999526</td>\n",
       "      <td>97230.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              precision    recall  f1-score       support\n",
       "0              0.999568  0.999935  0.999752  92600.000000\n",
       "1              0.998695  0.991361  0.995014   4630.000000\n",
       "accuracy       0.999527  0.999527  0.999527      0.999527\n",
       "macro avg      0.999131  0.995648  0.997383  97230.000000\n",
       "weighted avg   0.999527  0.999527  0.999526  97230.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "097f0042ac0d411882dc774a6ca638b0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/135 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Group 1a, step 1: 0.9704103671706263\n",
      "Group 1a, step 2: 0.9688984881209504\n",
      "Group 1a, step 3: 0.968682505399568\n",
      "Group 1a, step 4: 0.968682505399568\n",
      "Group 1a, step 5: 0.9684665226781858\n",
      "Group 1a, step 6: 0.9684665226781858\n",
      "Group 1a, step 7: 0.9684665226781858\n",
      "Group 1a, step 8: 0.9684665226781858\n",
      "Group 1a, step 9: 0.9682505399568034\n",
      "Group 1b, step 1: 0.9909287257019438\n",
      "Group 1b, step 2: 0.9909287257019438\n",
      "Group 1b, step 3: 0.9909287257019438\n",
      "Group 1b, step 4: 0.9909287257019438\n",
      "Group 1b, step 5: 0.9907127429805616\n",
      "Group 1b, step 6: 0.9907127429805616\n",
      "Group 1b, step 7: 0.9904967602591793\n",
      "Group 1b, step 8: 0.9894168466522678\n",
      "Group 1b, step 9: 0.9892008639308856\n",
      "Group 1c, step 1: 0.9909287257019438\n",
      "Group 1c, step 2: 0.9909287257019438\n",
      "Group 1c, step 3: 0.9909287257019438\n",
      "Group 1c, step 4: 0.9909287257019438\n",
      "Group 1c, step 5: 0.9907127429805616\n",
      "Group 1c, step 6: 0.9907127429805616\n",
      "Group 1c, step 7: 0.9907127429805616\n",
      "Group 1c, step 8: 0.9909287257019438\n",
      "Group 1c, step 9: 0.9904967602591793\n",
      "Group 1d, step 1: 0.9920086393088553\n",
      "Group 1d, step 2: 0.9911447084233261\n",
      "Group 1d, step 3: 0.9915766738660907\n",
      "Group 1d, step 4: 0.9904967602591793\n",
      "Group 1d, step 5: 0.9907127429805616\n",
      "Group 1d, step 6: 0.9907127429805616\n",
      "Group 1d, step 7: 0.9900647948164147\n",
      "Group 1d, step 8: 0.9898488120950324\n",
      "Group 1d, step 9: 0.9904967602591793\n",
      "Group 2a, step 1: 0.9691144708423326\n",
      "Group 2a, step 2: 0.9658747300215983\n",
      "Group 2a, step 3: 0.9658747300215983\n",
      "Group 2a, step 4: 0.9654427645788337\n",
      "Group 2a, step 5: 0.9652267818574514\n",
      "Group 2a, step 6: 0.9650107991360691\n",
      "Group 2a, step 7: 0.9643628509719222\n",
      "Group 2a, step 8: 0.963498920086393\n",
      "Group 2a, step 9: 0.9632829373650108\n",
      "Group 2b, step 1: 0.9693304535637149\n",
      "Group 2b, step 2: 0.9665226781857451\n",
      "Group 2b, step 3: 0.9658747300215983\n",
      "Group 2b, step 4: 0.9645788336933045\n",
      "Group 2b, step 5: 0.9643628509719222\n",
      "Group 2b, step 6: 0.9643628509719222\n",
      "Group 2b, step 7: 0.9643628509719222\n",
      "Group 2b, step 8: 0.963498920086393\n",
      "Group 2b, step 9: 0.963498920086393\n",
      "Group 2c, step 1: 0.9714902807775379\n",
      "Group 2c, step 2: 0.9693304535637149\n",
      "Group 2c, step 3: 0.9643628509719222\n",
      "Group 2c, step 4: 0.9580993520518358\n",
      "Group 2c, step 5: 0.9578833693304536\n",
      "Group 2c, step 6: 0.9578833693304536\n",
      "Group 2c, step 7: 0.9578833693304536\n",
      "Group 2c, step 8: 0.9576673866090712\n",
      "Group 2c, step 9: 0.9570194384449244\n",
      "Group 2d, step 1: 0.9904967602591793\n",
      "Group 2d, step 2: 0.9907127429805616\n",
      "Group 2d, step 3: 0.9892008639308856\n",
      "Group 2d, step 4: 0.9829373650107991\n",
      "Group 2d, step 5: 0.9775377969762419\n",
      "Group 2d, step 6: 0.9768898488120951\n",
      "Group 2d, step 7: 0.9792656587473002\n",
      "Group 2d, step 8: 0.9764578833693305\n",
      "Group 2d, step 9: 0.9792656587473002\n",
      "Group 2e, step 1: 0.9909287257019438\n",
      "Group 2e, step 2: 0.9909287257019438\n",
      "Group 2e, step 3: 0.9907127429805616\n",
      "Group 2e, step 4: 0.9907127429805616\n",
      "Group 2e, step 5: 0.990280777537797\n",
      "Group 2e, step 6: 0.9898488120950324\n",
      "Group 2e, step 7: 0.9892008639308856\n",
      "Group 2e, step 8: 0.988768898488121\n",
      "Group 2e, step 9: 0.988768898488121\n",
      "Group 2f, step 1: 0.9911447084233261\n",
      "Group 2f, step 2: 0.9909287257019438\n",
      "Group 2f, step 3: 0.9907127429805616\n",
      "Group 2f, step 4: 0.9872570194384449\n",
      "Group 2f, step 5: 0.9838012958963283\n",
      "Group 2f, step 6: 0.9833693304535637\n",
      "Group 2f, step 7: 0.9846652267818574\n",
      "Group 2f, step 8: 0.9853131749460043\n",
      "Group 2f, step 9: 0.9853131749460043\n",
      "Group 3a, step 1: 0.9678185745140389\n",
      "Group 3a, step 2: 0.9652267818574514\n",
      "Group 3a, step 3: 0.9645788336933045\n",
      "Group 3a, step 4: 0.9639308855291576\n",
      "Group 3a, step 5: 0.9613390928725702\n",
      "Group 3a, step 6: 0.9611231101511879\n",
      "Group 3a, step 7: 0.9593952483801296\n",
      "Group 3a, step 8: 0.9591792656587473\n",
      "Group 3a, step 9: 0.9088552915766739\n",
      "Group 3b, step 1: 0.9688984881209504\n",
      "Group 3b, step 2: 0.96414686825054\n",
      "Group 3b, step 3: 0.9591792656587473\n",
      "Group 3b, step 4: 0.9557235421166307\n",
      "Group 3b, step 5: 0.9548596112311015\n",
      "Group 3b, step 6: 0.9544276457883369\n",
      "Group 3b, step 7: 0.9542116630669546\n",
      "Group 3b, step 8: 0.9535637149028078\n",
      "Group 3b, step 9: 0.9518358531317495\n",
      "Group 3c, step 1: 0.9693304535637149\n",
      "Group 3c, step 2: 0.9660907127429805\n",
      "Group 3c, step 3: 0.9606911447084233\n",
      "Group 3c, step 4: 0.9550755939524838\n",
      "Group 3c, step 5: 0.9548596112311015\n",
      "Group 3c, step 6: 0.9548596112311015\n",
      "Group 3c, step 7: 0.9544276457883369\n",
      "Group 3c, step 8: 0.9539956803455724\n",
      "Group 3c, step 9: 0.9533477321814254\n",
      "Group 3d, step 1: 0.990280777537797\n",
      "Group 3d, step 2: 0.990280777537797\n",
      "Group 3d, step 3: 0.9876889848812095\n",
      "Group 3d, step 4: 0.9606911447084233\n",
      "Group 3d, step 5: 0.9580993520518358\n",
      "Group 3d, step 6: 0.957451403887689\n",
      "Group 3d, step 7: 0.9570194384449244\n",
      "Group 3d, step 8: 0.9565874730021598\n",
      "Group 3d, step 9: 0.9745140388768898\n",
      "Group 4a, step 1: 0.9673866090712743\n",
      "Group 4a, step 2: 0.963498920086393\n",
      "Group 4a, step 3: 0.9580993520518358\n",
      "Group 4a, step 4: 0.9550755939524838\n",
      "Group 4a, step 5: 0.9548596112311015\n",
      "Group 4a, step 6: 0.9514038876889849\n",
      "Group 4a, step 7: 0.9498920086393089\n",
      "Group 4a, step 8: 0.9494600431965443\n",
      "Group 4a, step 9: 0.8861771058315335\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1a</th>\n",
       "      <td>0.970410</td>\n",
       "      <td>0.968898</td>\n",
       "      <td>0.968683</td>\n",
       "      <td>0.968683</td>\n",
       "      <td>0.968467</td>\n",
       "      <td>0.968467</td>\n",
       "      <td>0.968467</td>\n",
       "      <td>0.968467</td>\n",
       "      <td>0.968251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1b</th>\n",
       "      <td>0.990929</td>\n",
       "      <td>0.990929</td>\n",
       "      <td>0.990929</td>\n",
       "      <td>0.990929</td>\n",
       "      <td>0.990713</td>\n",
       "      <td>0.990713</td>\n",
       "      <td>0.990497</td>\n",
       "      <td>0.989417</td>\n",
       "      <td>0.989201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1c</th>\n",
       "      <td>0.990929</td>\n",
       "      <td>0.990929</td>\n",
       "      <td>0.990929</td>\n",
       "      <td>0.990929</td>\n",
       "      <td>0.990713</td>\n",
       "      <td>0.990713</td>\n",
       "      <td>0.990713</td>\n",
       "      <td>0.990929</td>\n",
       "      <td>0.990497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1d</th>\n",
       "      <td>0.992009</td>\n",
       "      <td>0.991145</td>\n",
       "      <td>0.991577</td>\n",
       "      <td>0.990497</td>\n",
       "      <td>0.990713</td>\n",
       "      <td>0.990713</td>\n",
       "      <td>0.990065</td>\n",
       "      <td>0.989849</td>\n",
       "      <td>0.990497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2a</th>\n",
       "      <td>0.969114</td>\n",
       "      <td>0.965875</td>\n",
       "      <td>0.965875</td>\n",
       "      <td>0.965443</td>\n",
       "      <td>0.965227</td>\n",
       "      <td>0.965011</td>\n",
       "      <td>0.964363</td>\n",
       "      <td>0.963499</td>\n",
       "      <td>0.963283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2b</th>\n",
       "      <td>0.969330</td>\n",
       "      <td>0.966523</td>\n",
       "      <td>0.965875</td>\n",
       "      <td>0.964579</td>\n",
       "      <td>0.964363</td>\n",
       "      <td>0.964363</td>\n",
       "      <td>0.964363</td>\n",
       "      <td>0.963499</td>\n",
       "      <td>0.963499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2c</th>\n",
       "      <td>0.971490</td>\n",
       "      <td>0.969330</td>\n",
       "      <td>0.964363</td>\n",
       "      <td>0.958099</td>\n",
       "      <td>0.957883</td>\n",
       "      <td>0.957883</td>\n",
       "      <td>0.957883</td>\n",
       "      <td>0.957667</td>\n",
       "      <td>0.957019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2d</th>\n",
       "      <td>0.990497</td>\n",
       "      <td>0.990713</td>\n",
       "      <td>0.989201</td>\n",
       "      <td>0.982937</td>\n",
       "      <td>0.977538</td>\n",
       "      <td>0.976890</td>\n",
       "      <td>0.979266</td>\n",
       "      <td>0.976458</td>\n",
       "      <td>0.979266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2e</th>\n",
       "      <td>0.990929</td>\n",
       "      <td>0.990929</td>\n",
       "      <td>0.990713</td>\n",
       "      <td>0.990713</td>\n",
       "      <td>0.990281</td>\n",
       "      <td>0.989849</td>\n",
       "      <td>0.989201</td>\n",
       "      <td>0.988769</td>\n",
       "      <td>0.988769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2f</th>\n",
       "      <td>0.991145</td>\n",
       "      <td>0.990929</td>\n",
       "      <td>0.990713</td>\n",
       "      <td>0.987257</td>\n",
       "      <td>0.983801</td>\n",
       "      <td>0.983369</td>\n",
       "      <td>0.984665</td>\n",
       "      <td>0.985313</td>\n",
       "      <td>0.985313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3a</th>\n",
       "      <td>0.967819</td>\n",
       "      <td>0.965227</td>\n",
       "      <td>0.964579</td>\n",
       "      <td>0.963931</td>\n",
       "      <td>0.961339</td>\n",
       "      <td>0.961123</td>\n",
       "      <td>0.959395</td>\n",
       "      <td>0.959179</td>\n",
       "      <td>0.908855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3b</th>\n",
       "      <td>0.968898</td>\n",
       "      <td>0.964147</td>\n",
       "      <td>0.959179</td>\n",
       "      <td>0.955724</td>\n",
       "      <td>0.954860</td>\n",
       "      <td>0.954428</td>\n",
       "      <td>0.954212</td>\n",
       "      <td>0.953564</td>\n",
       "      <td>0.951836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3c</th>\n",
       "      <td>0.969330</td>\n",
       "      <td>0.966091</td>\n",
       "      <td>0.960691</td>\n",
       "      <td>0.955076</td>\n",
       "      <td>0.954860</td>\n",
       "      <td>0.954860</td>\n",
       "      <td>0.954428</td>\n",
       "      <td>0.953996</td>\n",
       "      <td>0.953348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3d</th>\n",
       "      <td>0.990281</td>\n",
       "      <td>0.990281</td>\n",
       "      <td>0.987689</td>\n",
       "      <td>0.960691</td>\n",
       "      <td>0.958099</td>\n",
       "      <td>0.957451</td>\n",
       "      <td>0.957019</td>\n",
       "      <td>0.956587</td>\n",
       "      <td>0.974514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4a</th>\n",
       "      <td>0.967387</td>\n",
       "      <td>0.963499</td>\n",
       "      <td>0.958099</td>\n",
       "      <td>0.955076</td>\n",
       "      <td>0.954860</td>\n",
       "      <td>0.951404</td>\n",
       "      <td>0.949892</td>\n",
       "      <td>0.949460</td>\n",
       "      <td>0.886177</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2         3         4         5         6  \\\n",
       "1a  0.970410  0.968898  0.968683  0.968683  0.968467  0.968467  0.968467   \n",
       "1b  0.990929  0.990929  0.990929  0.990929  0.990713  0.990713  0.990497   \n",
       "1c  0.990929  0.990929  0.990929  0.990929  0.990713  0.990713  0.990713   \n",
       "1d  0.992009  0.991145  0.991577  0.990497  0.990713  0.990713  0.990065   \n",
       "2a  0.969114  0.965875  0.965875  0.965443  0.965227  0.965011  0.964363   \n",
       "2b  0.969330  0.966523  0.965875  0.964579  0.964363  0.964363  0.964363   \n",
       "2c  0.971490  0.969330  0.964363  0.958099  0.957883  0.957883  0.957883   \n",
       "2d  0.990497  0.990713  0.989201  0.982937  0.977538  0.976890  0.979266   \n",
       "2e  0.990929  0.990929  0.990713  0.990713  0.990281  0.989849  0.989201   \n",
       "2f  0.991145  0.990929  0.990713  0.987257  0.983801  0.983369  0.984665   \n",
       "3a  0.967819  0.965227  0.964579  0.963931  0.961339  0.961123  0.959395   \n",
       "3b  0.968898  0.964147  0.959179  0.955724  0.954860  0.954428  0.954212   \n",
       "3c  0.969330  0.966091  0.960691  0.955076  0.954860  0.954860  0.954428   \n",
       "3d  0.990281  0.990281  0.987689  0.960691  0.958099  0.957451  0.957019   \n",
       "4a  0.967387  0.963499  0.958099  0.955076  0.954860  0.951404  0.949892   \n",
       "\n",
       "           7         8  \n",
       "1a  0.968467  0.968251  \n",
       "1b  0.989417  0.989201  \n",
       "1c  0.990929  0.990497  \n",
       "1d  0.989849  0.990497  \n",
       "2a  0.963499  0.963283  \n",
       "2b  0.963499  0.963499  \n",
       "2c  0.957667  0.957019  \n",
       "2d  0.976458  0.979266  \n",
       "2e  0.988769  0.988769  \n",
       "2f  0.985313  0.985313  \n",
       "3a  0.959179  0.908855  \n",
       "3b  0.953564  0.951836  \n",
       "3c  0.953996  0.953348  \n",
       "3d  0.956587  0.974514  \n",
       "4a  0.949460  0.886177  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.999772</td>\n",
       "      <td>0.999865</td>\n",
       "      <td>0.999818</td>\n",
       "      <td>96350.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.997297</td>\n",
       "      <td>0.995434</td>\n",
       "      <td>0.996364</td>\n",
       "      <td>4818.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.999654</td>\n",
       "      <td>0.999654</td>\n",
       "      <td>0.999654</td>\n",
       "      <td>0.999654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>macro avg</th>\n",
       "      <td>0.998534</td>\n",
       "      <td>0.997649</td>\n",
       "      <td>0.998091</td>\n",
       "      <td>101168.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weighted avg</th>\n",
       "      <td>0.999654</td>\n",
       "      <td>0.999654</td>\n",
       "      <td>0.999654</td>\n",
       "      <td>101168.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              precision    recall  f1-score        support\n",
       "0              0.999772  0.999865  0.999818   96350.000000\n",
       "1              0.997297  0.995434  0.996364    4818.000000\n",
       "accuracy       0.999654  0.999654  0.999654       0.999654\n",
       "macro avg      0.998534  0.997649  0.998091  101168.000000\n",
       "weighted avg   0.999654  0.999654  0.999654  101168.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f0b1335e8614d17826ba2b679af7814",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/135 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Group 1a, step 1: 0.9933582399335824\n",
      "Group 1a, step 2: 0.9923204649232047\n",
      "Group 1a, step 3: 0.9804898298048983\n",
      "Group 1a, step 4: 0.9889995848899958\n",
      "Group 1a, step 5: 0.9923204649232047\n",
      "Group 1a, step 6: 0.9921129099211291\n",
      "Group 1a, step 7: 0.9925280199252802\n",
      "Group 1a, step 8: 0.9906600249066002\n",
      "Group 1a, step 9: 0.9850560398505604\n",
      "Group 1b, step 1: 0.9954337899543378\n",
      "Group 1b, step 2: 0.9954337899543378\n",
      "Group 1b, step 3: 0.981527604815276\n",
      "Group 1b, step 4: 0.9597343295973433\n",
      "Group 1b, step 5: 0.9541303445413034\n",
      "Group 1b, step 6: 0.9530925695309257\n",
      "Group 1b, step 7: 0.9970942299709423\n",
      "Group 1b, step 8: 0.9950186799501868\n",
      "Group 1b, step 9: 0.9846409298464093\n",
      "Group 1c, step 1: 0.9954337899543378\n",
      "Group 1c, step 2: 0.9954337899543378\n",
      "Group 1c, step 3: 0.9954337899543378\n",
      "Group 1c, step 4: 0.9954337899543378\n",
      "Group 1c, step 5: 0.9927355749273558\n",
      "Group 1c, step 6: 0.9927355749273558\n",
      "Group 1c, step 7: 0.9908675799086758\n",
      "Group 1c, step 8: 0.9844333748443338\n",
      "Group 1c, step 9: 0.9802822748028227\n",
      "Group 1d, step 1: 0.9954337899543378\n",
      "Group 1d, step 2: 0.9952262349522624\n",
      "Group 1d, step 3: 0.9948111249481113\n",
      "Group 1d, step 4: 0.9948111249481113\n",
      "Group 1d, step 5: 0.9946035699460357\n",
      "Group 1d, step 6: 0.9946035699460357\n",
      "Group 1d, step 7: 0.9946035699460357\n",
      "Group 1d, step 8: 0.9946035699460357\n",
      "Group 1d, step 9: 0.9946035699460357\n",
      "Group 2a, step 1: 0.9933582399335824\n",
      "Group 2a, step 2: 0.9931506849315068\n",
      "Group 2a, step 3: 0.9696969696969697\n",
      "Group 2a, step 4: 0.9559983395599834\n",
      "Group 2a, step 5: 0.9535076795350768\n",
      "Group 2a, step 6: 0.9582814445828145\n",
      "Group 2a, step 7: 0.9819427148194272\n",
      "Group 2a, step 8: 0.8877127438771274\n",
      "Group 2a, step 9: 0.748650892486509\n",
      "Group 2b, step 1: 0.9933582399335824\n",
      "Group 2b, step 2: 0.9933582399335824\n",
      "Group 2b, step 3: 0.9798671647986716\n",
      "Group 2b, step 4: 0.9885844748858448\n",
      "Group 2b, step 5: 0.9896222498962225\n",
      "Group 2b, step 6: 0.9850560398505604\n",
      "Group 2b, step 7: 0.9337899543378996\n",
      "Group 2b, step 8: 0.798879202988792\n",
      "Group 2b, step 9: 0.7801992528019925\n",
      "Group 2c, step 1: 0.9954337899543378\n",
      "Group 2c, step 2: 0.9954337899543378\n",
      "Group 2c, step 3: 0.9800747198007472\n",
      "Group 2c, step 4: 0.9840182648401826\n",
      "Group 2c, step 5: 0.9761311747613117\n",
      "Group 2c, step 6: 0.962640099626401\n",
      "Group 2c, step 7: 0.7552926525529265\n",
      "Group 2c, step 8: 0.5060190950601909\n",
      "Group 2c, step 9: 0.49024491490244915\n",
      "Group 2d, step 1: 0.9954337899543378\n",
      "Group 2d, step 2: 0.9952262349522624\n",
      "Group 2d, step 3: 0.9458281444582815\n",
      "Group 2d, step 4: 0.9433374844333748\n",
      "Group 2d, step 5: 0.9416770444167705\n",
      "Group 2d, step 6: 0.9398090493980905\n",
      "Group 2d, step 7: 0.9518472395184724\n",
      "Group 2d, step 8: 0.9447903694479037\n",
      "Group 2d, step 9: 0.9375259443752595\n",
      "Group 2e, step 1: 0.9954337899543378\n",
      "Group 2e, step 2: 0.9954337899543378\n",
      "Group 2e, step 3: 0.9599418845994189\n",
      "Group 2e, step 4: 0.9553756745537567\n",
      "Group 2e, step 5: 0.9487339144873391\n",
      "Group 2e, step 6: 0.9485263594852636\n",
      "Group 2e, step 7: 0.9943960149439601\n",
      "Group 2e, step 8: 0.9840182648401826\n",
      "Group 2e, step 9: 0.9591116645911166\n",
      "Group 2f, step 1: 0.9954337899543378\n",
      "Group 2f, step 2: 0.9954337899543378\n",
      "Group 2f, step 3: 0.9707347447073474\n",
      "Group 2f, step 4: 0.9539227895392279\n",
      "Group 2f, step 5: 0.9433374844333748\n",
      "Group 2f, step 6: 0.9387712743877128\n",
      "Group 2f, step 7: 0.931714404317144\n",
      "Group 2f, step 8: 0.8835616438356164\n",
      "Group 2f, step 9: 0.9337899543378996\n",
      "Group 3a, step 1: 0.993565794935658\n",
      "Group 3a, step 2: 0.9931506849315068\n",
      "Group 3a, step 3: 0.9510170195101701\n",
      "Group 3a, step 4: 0.9537152345371523\n",
      "Group 3a, step 5: 0.9425072644250726\n",
      "Group 3a, step 6: 0.9373183893731839\n",
      "Group 3a, step 7: 0.7984640929846409\n",
      "Group 3a, step 8: 0.6903279369032793\n",
      "Group 3a, step 9: 0.35429638854296386\n",
      "Group 3b, step 1: 0.9956413449564134\n",
      "Group 3b, step 2: 0.9952262349522624\n",
      "Group 3b, step 3: 0.9362806143628062\n",
      "Group 3b, step 4: 0.9329597343295973\n",
      "Group 3b, step 5: 0.9178082191780822\n",
      "Group 3b, step 6: 0.8752594437525945\n",
      "Group 3b, step 7: 0.35699460356994606\n",
      "Group 3b, step 8: 0.2650477376504774\n",
      "Group 3b, step 9: 0.21585720215857201\n",
      "Group 3c, step 1: 0.9956413449564134\n",
      "Group 3c, step 2: 0.9954337899543378\n",
      "Group 3c, step 3: 0.9510170195101701\n",
      "Group 3c, step 4: 0.9323370693233707\n",
      "Group 3c, step 5: 0.8437110834371109\n",
      "Group 3c, step 6: 0.8202573682025737\n",
      "Group 3c, step 7: 0.4892071398920714\n",
      "Group 3c, step 8: 0.3318804483188045\n",
      "Group 3c, step 9: 0.24097135740971357\n",
      "Group 3d, step 1: 0.9954337899543378\n",
      "Group 3d, step 2: 0.9954337899543378\n",
      "Group 3d, step 3: 0.9435450394354504\n",
      "Group 3d, step 4: 0.9414694894146949\n",
      "Group 3d, step 5: 0.9350352843503529\n",
      "Group 3d, step 6: 0.9292237442922374\n",
      "Group 3d, step 7: 0.8823163138231631\n",
      "Group 3d, step 8: 0.8700705687007056\n",
      "Group 3d, step 9: 0.8841843088418431\n",
      "Group 4a, step 1: 0.9956413449564134\n",
      "Group 4a, step 2: 0.9952262349522624\n",
      "Group 4a, step 3: 0.9356579493565795\n",
      "Group 4a, step 4: 0.925280199252802\n",
      "Group 4a, step 5: 0.8210875882108759\n",
      "Group 4a, step 6: 0.7831050228310502\n",
      "Group 4a, step 7: 0.3011623080116231\n",
      "Group 4a, step 8: 0.21191365711913657\n",
      "Group 4a, step 9: 0.00228310502283105\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1a</th>\n",
       "      <td>0.993358</td>\n",
       "      <td>0.992320</td>\n",
       "      <td>0.980490</td>\n",
       "      <td>0.989000</td>\n",
       "      <td>0.992320</td>\n",
       "      <td>0.992113</td>\n",
       "      <td>0.992528</td>\n",
       "      <td>0.990660</td>\n",
       "      <td>0.985056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1b</th>\n",
       "      <td>0.995434</td>\n",
       "      <td>0.995434</td>\n",
       "      <td>0.981528</td>\n",
       "      <td>0.959734</td>\n",
       "      <td>0.954130</td>\n",
       "      <td>0.953093</td>\n",
       "      <td>0.997094</td>\n",
       "      <td>0.995019</td>\n",
       "      <td>0.984641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1c</th>\n",
       "      <td>0.995434</td>\n",
       "      <td>0.995434</td>\n",
       "      <td>0.995434</td>\n",
       "      <td>0.995434</td>\n",
       "      <td>0.992736</td>\n",
       "      <td>0.992736</td>\n",
       "      <td>0.990868</td>\n",
       "      <td>0.984433</td>\n",
       "      <td>0.980282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1d</th>\n",
       "      <td>0.995434</td>\n",
       "      <td>0.995226</td>\n",
       "      <td>0.994811</td>\n",
       "      <td>0.994811</td>\n",
       "      <td>0.994604</td>\n",
       "      <td>0.994604</td>\n",
       "      <td>0.994604</td>\n",
       "      <td>0.994604</td>\n",
       "      <td>0.994604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2a</th>\n",
       "      <td>0.993358</td>\n",
       "      <td>0.993151</td>\n",
       "      <td>0.969697</td>\n",
       "      <td>0.955998</td>\n",
       "      <td>0.953508</td>\n",
       "      <td>0.958281</td>\n",
       "      <td>0.981943</td>\n",
       "      <td>0.887713</td>\n",
       "      <td>0.748651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2b</th>\n",
       "      <td>0.993358</td>\n",
       "      <td>0.993358</td>\n",
       "      <td>0.979867</td>\n",
       "      <td>0.988584</td>\n",
       "      <td>0.989622</td>\n",
       "      <td>0.985056</td>\n",
       "      <td>0.933790</td>\n",
       "      <td>0.798879</td>\n",
       "      <td>0.780199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2c</th>\n",
       "      <td>0.995434</td>\n",
       "      <td>0.995434</td>\n",
       "      <td>0.980075</td>\n",
       "      <td>0.984018</td>\n",
       "      <td>0.976131</td>\n",
       "      <td>0.962640</td>\n",
       "      <td>0.755293</td>\n",
       "      <td>0.506019</td>\n",
       "      <td>0.490245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2d</th>\n",
       "      <td>0.995434</td>\n",
       "      <td>0.995226</td>\n",
       "      <td>0.945828</td>\n",
       "      <td>0.943337</td>\n",
       "      <td>0.941677</td>\n",
       "      <td>0.939809</td>\n",
       "      <td>0.951847</td>\n",
       "      <td>0.944790</td>\n",
       "      <td>0.937526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2e</th>\n",
       "      <td>0.995434</td>\n",
       "      <td>0.995434</td>\n",
       "      <td>0.959942</td>\n",
       "      <td>0.955376</td>\n",
       "      <td>0.948734</td>\n",
       "      <td>0.948526</td>\n",
       "      <td>0.994396</td>\n",
       "      <td>0.984018</td>\n",
       "      <td>0.959112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2f</th>\n",
       "      <td>0.995434</td>\n",
       "      <td>0.995434</td>\n",
       "      <td>0.970735</td>\n",
       "      <td>0.953923</td>\n",
       "      <td>0.943337</td>\n",
       "      <td>0.938771</td>\n",
       "      <td>0.931714</td>\n",
       "      <td>0.883562</td>\n",
       "      <td>0.933790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3a</th>\n",
       "      <td>0.993566</td>\n",
       "      <td>0.993151</td>\n",
       "      <td>0.951017</td>\n",
       "      <td>0.953715</td>\n",
       "      <td>0.942507</td>\n",
       "      <td>0.937318</td>\n",
       "      <td>0.798464</td>\n",
       "      <td>0.690328</td>\n",
       "      <td>0.354296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3b</th>\n",
       "      <td>0.995641</td>\n",
       "      <td>0.995226</td>\n",
       "      <td>0.936281</td>\n",
       "      <td>0.932960</td>\n",
       "      <td>0.917808</td>\n",
       "      <td>0.875259</td>\n",
       "      <td>0.356995</td>\n",
       "      <td>0.265048</td>\n",
       "      <td>0.215857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3c</th>\n",
       "      <td>0.995641</td>\n",
       "      <td>0.995434</td>\n",
       "      <td>0.951017</td>\n",
       "      <td>0.932337</td>\n",
       "      <td>0.843711</td>\n",
       "      <td>0.820257</td>\n",
       "      <td>0.489207</td>\n",
       "      <td>0.331880</td>\n",
       "      <td>0.240971</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3d</th>\n",
       "      <td>0.995434</td>\n",
       "      <td>0.995434</td>\n",
       "      <td>0.943545</td>\n",
       "      <td>0.941469</td>\n",
       "      <td>0.935035</td>\n",
       "      <td>0.929224</td>\n",
       "      <td>0.882316</td>\n",
       "      <td>0.870071</td>\n",
       "      <td>0.884184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4a</th>\n",
       "      <td>0.995641</td>\n",
       "      <td>0.995226</td>\n",
       "      <td>0.935658</td>\n",
       "      <td>0.925280</td>\n",
       "      <td>0.821088</td>\n",
       "      <td>0.783105</td>\n",
       "      <td>0.301162</td>\n",
       "      <td>0.211914</td>\n",
       "      <td>0.002283</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2         3         4         5         6  \\\n",
       "1a  0.993358  0.992320  0.980490  0.989000  0.992320  0.992113  0.992528   \n",
       "1b  0.995434  0.995434  0.981528  0.959734  0.954130  0.953093  0.997094   \n",
       "1c  0.995434  0.995434  0.995434  0.995434  0.992736  0.992736  0.990868   \n",
       "1d  0.995434  0.995226  0.994811  0.994811  0.994604  0.994604  0.994604   \n",
       "2a  0.993358  0.993151  0.969697  0.955998  0.953508  0.958281  0.981943   \n",
       "2b  0.993358  0.993358  0.979867  0.988584  0.989622  0.985056  0.933790   \n",
       "2c  0.995434  0.995434  0.980075  0.984018  0.976131  0.962640  0.755293   \n",
       "2d  0.995434  0.995226  0.945828  0.943337  0.941677  0.939809  0.951847   \n",
       "2e  0.995434  0.995434  0.959942  0.955376  0.948734  0.948526  0.994396   \n",
       "2f  0.995434  0.995434  0.970735  0.953923  0.943337  0.938771  0.931714   \n",
       "3a  0.993566  0.993151  0.951017  0.953715  0.942507  0.937318  0.798464   \n",
       "3b  0.995641  0.995226  0.936281  0.932960  0.917808  0.875259  0.356995   \n",
       "3c  0.995641  0.995434  0.951017  0.932337  0.843711  0.820257  0.489207   \n",
       "3d  0.995434  0.995434  0.943545  0.941469  0.935035  0.929224  0.882316   \n",
       "4a  0.995641  0.995226  0.935658  0.925280  0.821088  0.783105  0.301162   \n",
       "\n",
       "           7         8  \n",
       "1a  0.990660  0.985056  \n",
       "1b  0.995019  0.984641  \n",
       "1c  0.984433  0.980282  \n",
       "1d  0.994604  0.994604  \n",
       "2a  0.887713  0.748651  \n",
       "2b  0.798879  0.780199  \n",
       "2c  0.506019  0.490245  \n",
       "2d  0.944790  0.937526  \n",
       "2e  0.984018  0.959112  \n",
       "2f  0.883562  0.933790  \n",
       "3a  0.690328  0.354296  \n",
       "3b  0.265048  0.215857  \n",
       "3c  0.331880  0.240971  \n",
       "3d  0.870071  0.884184  \n",
       "4a  0.211914  0.002283  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.999355</td>\n",
       "      <td>0.999839</td>\n",
       "      <td>0.999596</td>\n",
       "      <td>99100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.996739</td>\n",
       "      <td>0.987084</td>\n",
       "      <td>0.991888</td>\n",
       "      <td>4955.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.999231</td>\n",
       "      <td>0.999231</td>\n",
       "      <td>0.999231</td>\n",
       "      <td>0.999231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>macro avg</th>\n",
       "      <td>0.998047</td>\n",
       "      <td>0.993461</td>\n",
       "      <td>0.995742</td>\n",
       "      <td>104055.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weighted avg</th>\n",
       "      <td>0.999230</td>\n",
       "      <td>0.999231</td>\n",
       "      <td>0.999229</td>\n",
       "      <td>104055.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              precision    recall  f1-score        support\n",
       "0              0.999355  0.999839  0.999596   99100.000000\n",
       "1              0.996739  0.987084  0.991888    4955.000000\n",
       "accuracy       0.999231  0.999231  0.999231       0.999231\n",
       "macro avg      0.998047  0.993461  0.995742  104055.000000\n",
       "weighted avg   0.999230  0.999231  0.999229  104055.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ff89d90cf9f478599843a254de49fea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/135 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Group 1a, step 1: 0.9884964682139253\n",
      "Group 1a, step 2: 0.9884964682139253\n",
      "Group 1a, step 3: 0.9884964682139253\n",
      "Group 1a, step 4: 0.9884964682139253\n",
      "Group 1a, step 5: 0.9884964682139253\n",
      "Group 1a, step 6: 0.9884964682139253\n",
      "Group 1a, step 7: 0.9884964682139253\n",
      "Group 1a, step 8: 0.9884964682139253\n",
      "Group 1a, step 9: 0.9884964682139253\n",
      "Group 1b, step 1: 0.9870837537840566\n",
      "Group 1b, step 2: 0.9866801210898083\n",
      "Group 1b, step 3: 0.9864783047426842\n",
      "Group 1b, step 4: 0.9864783047426842\n",
      "Group 1b, step 5: 0.9856710393541877\n",
      "Group 1b, step 6: 0.8768920282542886\n",
      "Group 1b, step 7: 0.8591321897073663\n",
      "Group 1b, step 8: 0.6542885973763874\n",
      "Group 1b, step 9: 0.7218970736629667\n",
      "Group 1c, step 1: 0.9870837537840566\n",
      "Group 1c, step 2: 0.9870837537840566\n",
      "Group 1c, step 3: 0.9870837537840566\n",
      "Group 1c, step 4: 0.9870837537840566\n",
      "Group 1c, step 5: 0.9866801210898083\n",
      "Group 1c, step 6: 0.9866801210898083\n",
      "Group 1c, step 7: 0.98627648839556\n",
      "Group 1c, step 8: 0.98627648839556\n",
      "Group 1c, step 9: 0.98627648839556\n",
      "Group 1d, step 1: 0.9868819374369324\n",
      "Group 1d, step 2: 0.9864783047426842\n",
      "Group 1d, step 3: 0.9866801210898083\n",
      "Group 1d, step 4: 0.9860746720484359\n",
      "Group 1d, step 5: 0.9858728557013118\n",
      "Group 1d, step 6: 0.9858728557013118\n",
      "Group 1d, step 7: 0.9858728557013118\n",
      "Group 1d, step 8: 0.9858728557013118\n",
      "Group 1d, step 9: 0.9858728557013118\n",
      "Group 2a, step 1: 0.9884964682139253\n",
      "Group 2a, step 2: 0.9884964682139253\n",
      "Group 2a, step 3: 0.9882946518668012\n",
      "Group 2a, step 4: 0.9882946518668012\n",
      "Group 2a, step 5: 0.589505549949546\n",
      "Group 2a, step 6: 0.3146316851664985\n",
      "Group 2a, step 7: 0.09303733602421796\n",
      "Group 2a, step 8: 0.21453077699293643\n",
      "Group 2a, step 9: 0.02199798183652876\n",
      "Group 2b, step 1: 0.9884964682139253\n",
      "Group 2b, step 2: 0.9884964682139253\n",
      "Group 2b, step 3: 0.9884964682139253\n",
      "Group 2b, step 4: 0.9884964682139253\n",
      "Group 2b, step 5: 0.9880928355196771\n",
      "Group 2b, step 6: 0.987891019172553\n",
      "Group 2b, step 7: 0.9876892028254288\n",
      "Group 2b, step 8: 0.9874873864783047\n",
      "Group 2b, step 9: 0.9872855701311807\n",
      "Group 2c, step 1: 0.9828456104944501\n",
      "Group 2c, step 2: 0.7608476286579213\n",
      "Group 2c, step 3: 0.7551967709384461\n",
      "Group 2c, step 4: 0.7551967709384461\n",
      "Group 2c, step 5: 0.7547931382441978\n",
      "Group 2c, step 6: 0.7547931382441978\n",
      "Group 2c, step 7: 0.7547931382441978\n",
      "Group 2c, step 8: 0.7547931382441978\n",
      "Group 2c, step 9: 0.7545913218970737\n",
      "Group 2d, step 1: 0.9864783047426842\n",
      "Group 2d, step 2: 0.9850655903128154\n",
      "Group 2d, step 3: 0.9846619576185671\n",
      "Group 2d, step 4: 0.977598385469223\n",
      "Group 2d, step 5: 0.8970736629667003\n",
      "Group 2d, step 6: 0.7521695257315842\n",
      "Group 2d, step 7: 0.7987891019172553\n",
      "Group 2d, step 8: 0.686175580221998\n",
      "Group 2d, step 9: 0.3927346115035318\n",
      "Group 2e, step 1: 0.9870837537840566\n",
      "Group 2e, step 2: 0.9866801210898083\n",
      "Group 2e, step 3: 0.9864783047426842\n",
      "Group 2e, step 4: 0.9864783047426842\n",
      "Group 2e, step 5: 0.9683148335015136\n",
      "Group 2e, step 6: 0.865590312815338\n",
      "Group 2e, step 7: 0.7784056508577195\n",
      "Group 2e, step 8: 0.6078708375378405\n",
      "Group 2e, step 9: 0.7166498486377396\n",
      "Group 2f, step 1: 0.9864783047426842\n",
      "Group 2f, step 2: 0.9856710393541877\n",
      "Group 2f, step 3: 0.9858728557013118\n",
      "Group 2f, step 4: 0.9784056508577195\n",
      "Group 2f, step 5: 0.9778002018163471\n",
      "Group 2f, step 6: 0.9778002018163471\n",
      "Group 2f, step 7: 0.9790110998990919\n",
      "Group 2f, step 8: 0.9788092835519677\n",
      "Group 2f, step 9: 0.9790110998990919\n",
      "Group 3a, step 1: 0.9884964682139253\n",
      "Group 3a, step 2: 0.9884964682139253\n",
      "Group 3a, step 3: 0.9882946518668012\n",
      "Group 3a, step 4: 0.987891019172553\n",
      "Group 3a, step 5: 0.558627648839556\n",
      "Group 3a, step 6: 0.30978809283551967\n",
      "Group 3a, step 7: 0.08113017154389505\n",
      "Group 3a, step 8: 0.08960645812310797\n",
      "Group 3a, step 9: 0.042179616548940466\n",
      "Group 3b, step 1: 0.7551967709384461\n",
      "Group 3b, step 2: 0.7551967709384461\n",
      "Group 3b, step 3: 0.7549949545913219\n",
      "Group 3b, step 4: 0.7549949545913219\n",
      "Group 3b, step 5: 0.15418768920282544\n",
      "Group 3b, step 6: 0.05893037336024218\n",
      "Group 3b, step 7: 0.03168516649848638\n",
      "Group 3b, step 8: 0.04399596367305752\n",
      "Group 3b, step 9: 0.010696266397578204\n",
      "Group 3c, step 1: 0.7551967709384461\n",
      "Group 3c, step 2: 0.7608476286579213\n",
      "Group 3c, step 3: 0.7551967709384461\n",
      "Group 3c, step 4: 0.7547931382441978\n",
      "Group 3c, step 5: 0.7547931382441978\n",
      "Group 3c, step 6: 0.7545913218970737\n",
      "Group 3c, step 7: 0.7545913218970737\n",
      "Group 3c, step 8: 0.751765893037336\n",
      "Group 3c, step 9: 0.7513622603430878\n",
      "Group 3d, step 1: 0.9864783047426842\n",
      "Group 3d, step 2: 0.9842583249243189\n",
      "Group 3d, step 3: 0.9838546922300706\n",
      "Group 3d, step 4: 0.9773965691220989\n",
      "Group 3d, step 5: 0.8958627648839556\n",
      "Group 3d, step 6: 0.7963673057517658\n",
      "Group 3d, step 7: 0.5493440968718466\n",
      "Group 3d, step 8: 0.5790110998990918\n",
      "Group 3d, step 9: 0.669021190716448\n",
      "Group 4a, step 1: 0.7551967709384461\n",
      "Group 4a, step 2: 0.7551967709384461\n",
      "Group 4a, step 3: 0.7549949545913219\n",
      "Group 4a, step 4: 0.7545913218970737\n",
      "Group 4a, step 5: 0.12613521695257315\n",
      "Group 4a, step 6: 0.030272452068617558\n",
      "Group 4a, step 7: 0.01089808274470232\n",
      "Group 4a, step 8: 0.003834510595358224\n",
      "Group 4a, step 9: 0.0006054490413723512\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1a</th>\n",
       "      <td>0.988496</td>\n",
       "      <td>0.988496</td>\n",
       "      <td>0.988496</td>\n",
       "      <td>0.988496</td>\n",
       "      <td>0.988496</td>\n",
       "      <td>0.988496</td>\n",
       "      <td>0.988496</td>\n",
       "      <td>0.988496</td>\n",
       "      <td>0.988496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1b</th>\n",
       "      <td>0.987084</td>\n",
       "      <td>0.986680</td>\n",
       "      <td>0.986478</td>\n",
       "      <td>0.986478</td>\n",
       "      <td>0.985671</td>\n",
       "      <td>0.876892</td>\n",
       "      <td>0.859132</td>\n",
       "      <td>0.654289</td>\n",
       "      <td>0.721897</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1c</th>\n",
       "      <td>0.987084</td>\n",
       "      <td>0.987084</td>\n",
       "      <td>0.987084</td>\n",
       "      <td>0.987084</td>\n",
       "      <td>0.986680</td>\n",
       "      <td>0.986680</td>\n",
       "      <td>0.986276</td>\n",
       "      <td>0.986276</td>\n",
       "      <td>0.986276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1d</th>\n",
       "      <td>0.986882</td>\n",
       "      <td>0.986478</td>\n",
       "      <td>0.986680</td>\n",
       "      <td>0.986075</td>\n",
       "      <td>0.985873</td>\n",
       "      <td>0.985873</td>\n",
       "      <td>0.985873</td>\n",
       "      <td>0.985873</td>\n",
       "      <td>0.985873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2a</th>\n",
       "      <td>0.988496</td>\n",
       "      <td>0.988496</td>\n",
       "      <td>0.988295</td>\n",
       "      <td>0.988295</td>\n",
       "      <td>0.589506</td>\n",
       "      <td>0.314632</td>\n",
       "      <td>0.093037</td>\n",
       "      <td>0.214531</td>\n",
       "      <td>0.021998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2b</th>\n",
       "      <td>0.988496</td>\n",
       "      <td>0.988496</td>\n",
       "      <td>0.988496</td>\n",
       "      <td>0.988496</td>\n",
       "      <td>0.988093</td>\n",
       "      <td>0.987891</td>\n",
       "      <td>0.987689</td>\n",
       "      <td>0.987487</td>\n",
       "      <td>0.987286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2c</th>\n",
       "      <td>0.982846</td>\n",
       "      <td>0.760848</td>\n",
       "      <td>0.755197</td>\n",
       "      <td>0.755197</td>\n",
       "      <td>0.754793</td>\n",
       "      <td>0.754793</td>\n",
       "      <td>0.754793</td>\n",
       "      <td>0.754793</td>\n",
       "      <td>0.754591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2d</th>\n",
       "      <td>0.986478</td>\n",
       "      <td>0.985066</td>\n",
       "      <td>0.984662</td>\n",
       "      <td>0.977598</td>\n",
       "      <td>0.897074</td>\n",
       "      <td>0.752170</td>\n",
       "      <td>0.798789</td>\n",
       "      <td>0.686176</td>\n",
       "      <td>0.392735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2e</th>\n",
       "      <td>0.987084</td>\n",
       "      <td>0.986680</td>\n",
       "      <td>0.986478</td>\n",
       "      <td>0.986478</td>\n",
       "      <td>0.968315</td>\n",
       "      <td>0.865590</td>\n",
       "      <td>0.778406</td>\n",
       "      <td>0.607871</td>\n",
       "      <td>0.716650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2f</th>\n",
       "      <td>0.986478</td>\n",
       "      <td>0.985671</td>\n",
       "      <td>0.985873</td>\n",
       "      <td>0.978406</td>\n",
       "      <td>0.977800</td>\n",
       "      <td>0.977800</td>\n",
       "      <td>0.979011</td>\n",
       "      <td>0.978809</td>\n",
       "      <td>0.979011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3a</th>\n",
       "      <td>0.988496</td>\n",
       "      <td>0.988496</td>\n",
       "      <td>0.988295</td>\n",
       "      <td>0.987891</td>\n",
       "      <td>0.558628</td>\n",
       "      <td>0.309788</td>\n",
       "      <td>0.081130</td>\n",
       "      <td>0.089606</td>\n",
       "      <td>0.042180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3b</th>\n",
       "      <td>0.755197</td>\n",
       "      <td>0.755197</td>\n",
       "      <td>0.754995</td>\n",
       "      <td>0.754995</td>\n",
       "      <td>0.154188</td>\n",
       "      <td>0.058930</td>\n",
       "      <td>0.031685</td>\n",
       "      <td>0.043996</td>\n",
       "      <td>0.010696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3c</th>\n",
       "      <td>0.755197</td>\n",
       "      <td>0.760848</td>\n",
       "      <td>0.755197</td>\n",
       "      <td>0.754793</td>\n",
       "      <td>0.754793</td>\n",
       "      <td>0.754591</td>\n",
       "      <td>0.754591</td>\n",
       "      <td>0.751766</td>\n",
       "      <td>0.751362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3d</th>\n",
       "      <td>0.986478</td>\n",
       "      <td>0.984258</td>\n",
       "      <td>0.983855</td>\n",
       "      <td>0.977397</td>\n",
       "      <td>0.895863</td>\n",
       "      <td>0.796367</td>\n",
       "      <td>0.549344</td>\n",
       "      <td>0.579011</td>\n",
       "      <td>0.669021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4a</th>\n",
       "      <td>0.755197</td>\n",
       "      <td>0.755197</td>\n",
       "      <td>0.754995</td>\n",
       "      <td>0.754591</td>\n",
       "      <td>0.126135</td>\n",
       "      <td>0.030272</td>\n",
       "      <td>0.010898</td>\n",
       "      <td>0.003835</td>\n",
       "      <td>0.000605</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2         3         4         5         6  \\\n",
       "1a  0.988496  0.988496  0.988496  0.988496  0.988496  0.988496  0.988496   \n",
       "1b  0.987084  0.986680  0.986478  0.986478  0.985671  0.876892  0.859132   \n",
       "1c  0.987084  0.987084  0.987084  0.987084  0.986680  0.986680  0.986276   \n",
       "1d  0.986882  0.986478  0.986680  0.986075  0.985873  0.985873  0.985873   \n",
       "2a  0.988496  0.988496  0.988295  0.988295  0.589506  0.314632  0.093037   \n",
       "2b  0.988496  0.988496  0.988496  0.988496  0.988093  0.987891  0.987689   \n",
       "2c  0.982846  0.760848  0.755197  0.755197  0.754793  0.754793  0.754793   \n",
       "2d  0.986478  0.985066  0.984662  0.977598  0.897074  0.752170  0.798789   \n",
       "2e  0.987084  0.986680  0.986478  0.986478  0.968315  0.865590  0.778406   \n",
       "2f  0.986478  0.985671  0.985873  0.978406  0.977800  0.977800  0.979011   \n",
       "3a  0.988496  0.988496  0.988295  0.987891  0.558628  0.309788  0.081130   \n",
       "3b  0.755197  0.755197  0.754995  0.754995  0.154188  0.058930  0.031685   \n",
       "3c  0.755197  0.760848  0.755197  0.754793  0.754793  0.754591  0.754591   \n",
       "3d  0.986478  0.984258  0.983855  0.977397  0.895863  0.796367  0.549344   \n",
       "4a  0.755197  0.755197  0.754995  0.754591  0.126135  0.030272  0.010898   \n",
       "\n",
       "           7         8  \n",
       "1a  0.988496  0.988496  \n",
       "1b  0.654289  0.721897  \n",
       "1c  0.986276  0.986276  \n",
       "1d  0.985873  0.985873  \n",
       "2a  0.214531  0.021998  \n",
       "2b  0.987487  0.987286  \n",
       "2c  0.754793  0.754591  \n",
       "2d  0.686176  0.392735  \n",
       "2e  0.607871  0.716650  \n",
       "2f  0.978809  0.979011  \n",
       "3a  0.089606  0.042180  \n",
       "3b  0.043996  0.010696  \n",
       "3c  0.751766  0.751362  \n",
       "3d  0.579011  0.669021  \n",
       "4a  0.003835  0.000605  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.996552</td>\n",
       "      <td>0.999656</td>\n",
       "      <td>0.998101</td>\n",
       "      <td>98870.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.992666</td>\n",
       "      <td>0.930825</td>\n",
       "      <td>0.960752</td>\n",
       "      <td>4944.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.996378</td>\n",
       "      <td>0.996378</td>\n",
       "      <td>0.996378</td>\n",
       "      <td>0.996378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>macro avg</th>\n",
       "      <td>0.994609</td>\n",
       "      <td>0.965241</td>\n",
       "      <td>0.979427</td>\n",
       "      <td>103814.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weighted avg</th>\n",
       "      <td>0.996367</td>\n",
       "      <td>0.996378</td>\n",
       "      <td>0.996323</td>\n",
       "      <td>103814.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              precision    recall  f1-score        support\n",
       "0              0.996552  0.999656  0.998101   98870.000000\n",
       "1              0.992666  0.930825  0.960752    4944.000000\n",
       "accuracy       0.996378  0.996378  0.996378       0.996378\n",
       "macro avg      0.994609  0.965241  0.979427  103814.000000\n",
       "weighted avg   0.996367  0.996378  0.996323  103814.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "faa5dbc0c0fd43f19f81fb398bcae3d2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/135 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Group 1a, step 1: 0.5989077669902912\n",
      "Group 1a, step 2: 0.5414644012944984\n",
      "Group 1a, step 3: 0.5414644012944984\n",
      "Group 1a, step 4: 0.5414644012944984\n",
      "Group 1a, step 5: 0.5414644012944984\n",
      "Group 1a, step 6: 0.5414644012944984\n",
      "Group 1a, step 7: 0.5414644012944984\n",
      "Group 1a, step 8: 0.5414644012944984\n",
      "Group 1a, step 9: 0.5414644012944984\n",
      "Group 1b, step 1: 0.9306229773462783\n",
      "Group 1b, step 2: 0.9306229773462783\n",
      "Group 1b, step 3: 0.9288025889967637\n",
      "Group 1b, step 4: 0.9288025889967637\n",
      "Group 1b, step 5: 0.9288025889967637\n",
      "Group 1b, step 6: 0.9275889967637541\n",
      "Group 1b, step 7: 0.9275889967637541\n",
      "Group 1b, step 8: 0.926779935275081\n",
      "Group 1b, step 9: 0.9259708737864077\n",
      "Group 1c, step 1: 0.9306229773462783\n",
      "Group 1c, step 2: 0.9306229773462783\n",
      "Group 1c, step 3: 0.9294093851132686\n",
      "Group 1c, step 4: 0.9294093851132686\n",
      "Group 1c, step 5: 0.9294093851132686\n",
      "Group 1c, step 6: 0.9294093851132686\n",
      "Group 1c, step 7: 0.9286003236245954\n",
      "Group 1c, step 8: 0.9286003236245954\n",
      "Group 1c, step 9: 0.9281957928802589\n",
      "Group 1d, step 1: 0.9263754045307443\n",
      "Group 1d, step 2: 0.9263754045307443\n",
      "Group 1d, step 3: 0.9251618122977346\n",
      "Group 1d, step 4: 0.9251618122977346\n",
      "Group 1d, step 5: 0.9251618122977346\n",
      "Group 1d, step 6: 0.9251618122977346\n",
      "Group 1d, step 7: 0.9251618122977346\n",
      "Group 1d, step 8: 0.9251618122977346\n",
      "Group 1d, step 9: 0.9251618122977346\n",
      "Group 2a, step 1: 0.0093042071197411\n",
      "Group 2a, step 2: 0.0093042071197411\n",
      "Group 2a, step 3: 0.0093042071197411\n",
      "Group 2a, step 4: 0.0\n",
      "Group 2a, step 5: 0.0\n",
      "Group 2a, step 6: 0.0\n",
      "Group 2a, step 7: 0.0\n",
      "Group 2a, step 8: 0.0\n",
      "Group 2a, step 9: 0.0\n",
      "Group 2b, step 1: 0.011529126213592233\n",
      "Group 2b, step 2: 0.011529126213592233\n",
      "Group 2b, step 3: 0.011529126213592233\n",
      "Group 2b, step 4: 0.0\n",
      "Group 2b, step 5: 0.0\n",
      "Group 2b, step 6: 0.0\n",
      "Group 2b, step 7: 0.0\n",
      "Group 2b, step 8: 0.0\n",
      "Group 2b, step 9: 0.0\n",
      "Group 2c, step 1: 0.0\n",
      "Group 2c, step 2: 0.0\n",
      "Group 2c, step 3: 0.0\n",
      "Group 2c, step 4: 0.0\n",
      "Group 2c, step 5: 0.0\n",
      "Group 2c, step 6: 0.0\n",
      "Group 2c, step 7: 0.0\n",
      "Group 2c, step 8: 0.0\n",
      "Group 2c, step 9: 0.0\n",
      "Group 2d, step 1: 0.9213187702265372\n",
      "Group 2d, step 2: 0.8535598705501618\n",
      "Group 2d, step 3: 0.7892394822006472\n",
      "Group 2d, step 4: 0.7892394822006472\n",
      "Group 2d, step 5: 0.7892394822006472\n",
      "Group 2d, step 6: 0.7892394822006472\n",
      "Group 2d, step 7: 0.7892394822006472\n",
      "Group 2d, step 8: 0.7892394822006472\n",
      "Group 2d, step 9: 0.7866100323624595\n",
      "Group 2e, step 1: 0.9306229773462783\n",
      "Group 2e, step 2: 0.9306229773462783\n",
      "Group 2e, step 3: 0.9283980582524272\n",
      "Group 2e, step 4: 0.9271844660194175\n",
      "Group 2e, step 5: 0.9263754045307443\n",
      "Group 2e, step 6: 0.9253640776699029\n",
      "Group 2e, step 7: 0.9251618122977346\n",
      "Group 2e, step 8: 0.9249595469255664\n",
      "Group 2e, step 9: 0.9247572815533981\n",
      "Group 2f, step 1: 0.9235436893203883\n",
      "Group 2f, step 2: 0.9235436893203883\n",
      "Group 2f, step 3: 0.8491100323624595\n",
      "Group 2f, step 4: 0.8491100323624595\n",
      "Group 2f, step 5: 0.8491100323624595\n",
      "Group 2f, step 6: 0.8491100323624595\n",
      "Group 2f, step 7: 0.8491100323624595\n",
      "Group 2f, step 8: 0.8491100323624595\n",
      "Group 2f, step 9: 0.8491100323624595\n",
      "Group 3a, step 1: 0.0093042071197411\n",
      "Group 3a, step 2: 0.0093042071197411\n",
      "Group 3a, step 3: 0.0\n",
      "Group 3a, step 4: 0.0\n",
      "Group 3a, step 5: 0.0\n",
      "Group 3a, step 6: 0.0\n",
      "Group 3a, step 7: 0.0\n",
      "Group 3a, step 8: 0.0\n",
      "Group 3a, step 9: 0.0\n",
      "Group 3b, step 1: 0.0\n",
      "Group 3b, step 2: 0.0\n",
      "Group 3b, step 3: 0.0\n",
      "Group 3b, step 4: 0.0\n",
      "Group 3b, step 5: 0.0\n",
      "Group 3b, step 6: 0.0\n",
      "Group 3b, step 7: 0.0\n",
      "Group 3b, step 8: 0.0\n",
      "Group 3b, step 9: 0.0\n",
      "Group 3c, step 1: 0.0\n",
      "Group 3c, step 2: 0.0\n",
      "Group 3c, step 3: 0.0\n",
      "Group 3c, step 4: 0.0\n",
      "Group 3c, step 5: 0.0\n",
      "Group 3c, step 6: 0.0\n",
      "Group 3c, step 7: 0.0\n",
      "Group 3c, step 8: 0.0\n",
      "Group 3c, step 9: 0.0\n",
      "Group 3d, step 1: 0.8519417475728155\n",
      "Group 3d, step 2: 0.8483009708737864\n",
      "Group 3d, step 3: 0.7892394822006472\n",
      "Group 3d, step 4: 0.7892394822006472\n",
      "Group 3d, step 5: 0.7892394822006472\n",
      "Group 3d, step 6: 0.7892394822006472\n",
      "Group 3d, step 7: 0.7874190938511327\n",
      "Group 3d, step 8: 0.7851941747572816\n",
      "Group 3d, step 9: 0.7831715210355987\n",
      "Group 4a, step 1: 0.0\n",
      "Group 4a, step 2: 0.0\n",
      "Group 4a, step 3: 0.0\n",
      "Group 4a, step 4: 0.0\n",
      "Group 4a, step 5: 0.0\n",
      "Group 4a, step 6: 0.0\n",
      "Group 4a, step 7: 0.0\n",
      "Group 4a, step 8: 0.0\n",
      "Group 4a, step 9: 0.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1a</th>\n",
       "      <td>0.598908</td>\n",
       "      <td>0.541464</td>\n",
       "      <td>0.541464</td>\n",
       "      <td>0.541464</td>\n",
       "      <td>0.541464</td>\n",
       "      <td>0.541464</td>\n",
       "      <td>0.541464</td>\n",
       "      <td>0.541464</td>\n",
       "      <td>0.541464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1b</th>\n",
       "      <td>0.930623</td>\n",
       "      <td>0.930623</td>\n",
       "      <td>0.928803</td>\n",
       "      <td>0.928803</td>\n",
       "      <td>0.928803</td>\n",
       "      <td>0.927589</td>\n",
       "      <td>0.927589</td>\n",
       "      <td>0.926780</td>\n",
       "      <td>0.925971</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1c</th>\n",
       "      <td>0.930623</td>\n",
       "      <td>0.930623</td>\n",
       "      <td>0.929409</td>\n",
       "      <td>0.929409</td>\n",
       "      <td>0.929409</td>\n",
       "      <td>0.929409</td>\n",
       "      <td>0.928600</td>\n",
       "      <td>0.928600</td>\n",
       "      <td>0.928196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1d</th>\n",
       "      <td>0.926375</td>\n",
       "      <td>0.926375</td>\n",
       "      <td>0.925162</td>\n",
       "      <td>0.925162</td>\n",
       "      <td>0.925162</td>\n",
       "      <td>0.925162</td>\n",
       "      <td>0.925162</td>\n",
       "      <td>0.925162</td>\n",
       "      <td>0.925162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2a</th>\n",
       "      <td>0.009304</td>\n",
       "      <td>0.009304</td>\n",
       "      <td>0.009304</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2b</th>\n",
       "      <td>0.011529</td>\n",
       "      <td>0.011529</td>\n",
       "      <td>0.011529</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2c</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2d</th>\n",
       "      <td>0.921319</td>\n",
       "      <td>0.853560</td>\n",
       "      <td>0.789239</td>\n",
       "      <td>0.789239</td>\n",
       "      <td>0.789239</td>\n",
       "      <td>0.789239</td>\n",
       "      <td>0.789239</td>\n",
       "      <td>0.789239</td>\n",
       "      <td>0.786610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2e</th>\n",
       "      <td>0.930623</td>\n",
       "      <td>0.930623</td>\n",
       "      <td>0.928398</td>\n",
       "      <td>0.927184</td>\n",
       "      <td>0.926375</td>\n",
       "      <td>0.925364</td>\n",
       "      <td>0.925162</td>\n",
       "      <td>0.924960</td>\n",
       "      <td>0.924757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2f</th>\n",
       "      <td>0.923544</td>\n",
       "      <td>0.923544</td>\n",
       "      <td>0.849110</td>\n",
       "      <td>0.849110</td>\n",
       "      <td>0.849110</td>\n",
       "      <td>0.849110</td>\n",
       "      <td>0.849110</td>\n",
       "      <td>0.849110</td>\n",
       "      <td>0.849110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3a</th>\n",
       "      <td>0.009304</td>\n",
       "      <td>0.009304</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3b</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3c</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3d</th>\n",
       "      <td>0.851942</td>\n",
       "      <td>0.848301</td>\n",
       "      <td>0.789239</td>\n",
       "      <td>0.789239</td>\n",
       "      <td>0.789239</td>\n",
       "      <td>0.789239</td>\n",
       "      <td>0.787419</td>\n",
       "      <td>0.785194</td>\n",
       "      <td>0.783172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4a</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2         3         4         5         6  \\\n",
       "1a  0.598908  0.541464  0.541464  0.541464  0.541464  0.541464  0.541464   \n",
       "1b  0.930623  0.930623  0.928803  0.928803  0.928803  0.927589  0.927589   \n",
       "1c  0.930623  0.930623  0.929409  0.929409  0.929409  0.929409  0.928600   \n",
       "1d  0.926375  0.926375  0.925162  0.925162  0.925162  0.925162  0.925162   \n",
       "2a  0.009304  0.009304  0.009304  0.000000  0.000000  0.000000  0.000000   \n",
       "2b  0.011529  0.011529  0.011529  0.000000  0.000000  0.000000  0.000000   \n",
       "2c  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "2d  0.921319  0.853560  0.789239  0.789239  0.789239  0.789239  0.789239   \n",
       "2e  0.930623  0.930623  0.928398  0.927184  0.926375  0.925364  0.925162   \n",
       "2f  0.923544  0.923544  0.849110  0.849110  0.849110  0.849110  0.849110   \n",
       "3a  0.009304  0.009304  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "3b  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "3c  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "3d  0.851942  0.848301  0.789239  0.789239  0.789239  0.789239  0.787419   \n",
       "4a  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "\n",
       "           7         8  \n",
       "1a  0.541464  0.541464  \n",
       "1b  0.926780  0.925971  \n",
       "1c  0.928600  0.928196  \n",
       "1d  0.925162  0.925162  \n",
       "2a  0.000000  0.000000  \n",
       "2b  0.000000  0.000000  \n",
       "2c  0.000000  0.000000  \n",
       "2d  0.789239  0.786610  \n",
       "2e  0.924960  0.924757  \n",
       "2f  0.849110  0.849110  \n",
       "3a  0.000000  0.000000  \n",
       "3b  0.000000  0.000000  \n",
       "3c  0.000000  0.000000  \n",
       "3d  0.785194  0.783172  \n",
       "4a  0.000000  0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.999876</td>\n",
       "      <td>0.999990</td>\n",
       "      <td>0.999933</td>\n",
       "      <td>97120.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.999794</td>\n",
       "      <td>0.997529</td>\n",
       "      <td>0.998660</td>\n",
       "      <td>4856.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.999873</td>\n",
       "      <td>0.999873</td>\n",
       "      <td>0.999873</td>\n",
       "      <td>0.999873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>macro avg</th>\n",
       "      <td>0.999835</td>\n",
       "      <td>0.998759</td>\n",
       "      <td>0.999297</td>\n",
       "      <td>101976.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weighted avg</th>\n",
       "      <td>0.999873</td>\n",
       "      <td>0.999873</td>\n",
       "      <td>0.999872</td>\n",
       "      <td>101976.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              precision    recall  f1-score        support\n",
       "0              0.999876  0.999990  0.999933   97120.000000\n",
       "1              0.999794  0.997529  0.998660    4856.000000\n",
       "accuracy       0.999873  0.999873  0.999873       0.999873\n",
       "macro avg      0.999835  0.998759  0.999297  101976.000000\n",
       "weighted avg   0.999873  0.999873  0.999872  101976.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34b707b9d9334f2082fec0f7983bcff3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/135 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Group 1a, step 1: 0.9967051070840197\n",
      "Group 1a, step 2: 0.9958813838550247\n",
      "Group 1a, step 3: 0.9958813838550247\n",
      "Group 1a, step 4: 0.9958813838550247\n",
      "Group 1a, step 5: 0.9958813838550247\n",
      "Group 1a, step 6: 0.9958813838550247\n",
      "Group 1a, step 7: 0.9958813838550247\n",
      "Group 1a, step 8: 0.9956754530477759\n",
      "Group 1a, step 9: 0.9956754530477759\n",
      "Group 1b, step 1: 0.9975288303130149\n",
      "Group 1b, step 2: 0.9975288303130149\n",
      "Group 1b, step 3: 0.9969110378912686\n",
      "Group 1b, step 4: 0.9969110378912686\n",
      "Group 1b, step 5: 0.9962932454695222\n",
      "Group 1b, step 6: 0.9954695222405272\n",
      "Group 1b, step 7: 0.9954695222405272\n",
      "Group 1b, step 8: 0.9948517298187809\n",
      "Group 1b, step 9: 0.9948517298187809\n",
      "Group 1c, step 1: 0.9975288303130149\n",
      "Group 1c, step 2: 0.9975288303130149\n",
      "Group 1c, step 3: 0.9975288303130149\n",
      "Group 1c, step 4: 0.9975288303130149\n",
      "Group 1c, step 5: 0.997322899505766\n",
      "Group 1c, step 6: 0.997322899505766\n",
      "Group 1c, step 7: 0.9971169686985173\n",
      "Group 1c, step 8: 0.9969110378912686\n",
      "Group 1c, step 9: 0.9969110378912686\n",
      "Group 1d, step 1: 0.9975288303130149\n",
      "Group 1d, step 2: 0.997322899505766\n",
      "Group 1d, step 3: 0.9969110378912686\n",
      "Group 1d, step 4: 0.9967051070840197\n",
      "Group 1d, step 5: 0.996499176276771\n",
      "Group 1d, step 6: 0.996499176276771\n",
      "Group 1d, step 7: 0.996499176276771\n",
      "Group 1d, step 8: 0.996499176276771\n",
      "Group 1d, step 9: 0.996499176276771\n",
      "Group 2a, step 1: 0.9967051070840197\n",
      "Group 2a, step 2: 0.9956754530477759\n",
      "Group 2a, step 3: 0.9956754530477759\n",
      "Group 2a, step 4: 0.9954695222405272\n",
      "Group 2a, step 5: 0.9950576606260296\n",
      "Group 2a, step 6: 0.9915568369028006\n",
      "Group 2a, step 7: 0.9915568369028006\n",
      "Group 2a, step 8: 0.9913509060955519\n",
      "Group 2a, step 9: 0.9911449752883031\n",
      "Group 2b, step 1: 0.9967051070840197\n",
      "Group 2b, step 2: 0.9958813838550247\n",
      "Group 2b, step 3: 0.9958813838550247\n",
      "Group 2b, step 4: 0.9956754530477759\n",
      "Group 2b, step 5: 0.9954695222405272\n",
      "Group 2b, step 6: 0.9952635914332785\n",
      "Group 2b, step 7: 0.9950576606260296\n",
      "Group 2b, step 8: 0.9950576606260296\n",
      "Group 2b, step 9: 0.9950576606260296\n",
      "Group 2c, step 1: 0.996499176276771\n",
      "Group 2c, step 2: 0.9880560131795717\n",
      "Group 2c, step 3: 0.32784184514003295\n",
      "Group 2c, step 4: 0.32784184514003295\n",
      "Group 2c, step 5: 0.32784184514003295\n",
      "Group 2c, step 6: 0.32784184514003295\n",
      "Group 2c, step 7: 0.32784184514003295\n",
      "Group 2c, step 8: 0.32763591433278416\n",
      "Group 2c, step 9: 0.364497528830313\n",
      "Group 2d, step 1: 0.9969110378912686\n",
      "Group 2d, step 2: 0.996499176276771\n",
      "Group 2d, step 3: 0.9952635914332785\n",
      "Group 2d, step 4: 0.9950576606260296\n",
      "Group 2d, step 5: 0.9948517298187809\n",
      "Group 2d, step 6: 0.9124794069192751\n",
      "Group 2d, step 7: 0.9122734761120264\n",
      "Group 2d, step 8: 0.9118616144975288\n",
      "Group 2d, step 9: 0.9349258649093904\n",
      "Group 2e, step 1: 0.9975288303130149\n",
      "Group 2e, step 2: 0.9975288303130149\n",
      "Group 2e, step 3: 0.9969110378912686\n",
      "Group 2e, step 4: 0.9962932454695222\n",
      "Group 2e, step 5: 0.9954695222405272\n",
      "Group 2e, step 6: 0.9950576606260296\n",
      "Group 2e, step 7: 0.9948517298187809\n",
      "Group 2e, step 8: 0.9946457990115322\n",
      "Group 2e, step 9: 0.9942339373970346\n",
      "Group 2f, step 1: 0.9969110378912686\n",
      "Group 2f, step 2: 0.996499176276771\n",
      "Group 2f, step 3: 0.996499176276771\n",
      "Group 2f, step 4: 0.9960873146622735\n",
      "Group 2f, step 5: 0.9950576606260296\n",
      "Group 2f, step 6: 0.9948517298187809\n",
      "Group 2f, step 7: 0.9948517298187809\n",
      "Group 2f, step 8: 0.9608731466227347\n",
      "Group 2f, step 9: 0.9608731466227347\n",
      "Group 3a, step 1: 0.9958813838550247\n",
      "Group 3a, step 2: 0.9956754530477759\n",
      "Group 3a, step 3: 0.9954695222405272\n",
      "Group 3a, step 4: 0.9954695222405272\n",
      "Group 3a, step 5: 0.9915568369028006\n",
      "Group 3a, step 6: 0.9913509060955519\n",
      "Group 3a, step 7: 0.9913509060955519\n",
      "Group 3a, step 8: 0.9911449752883031\n",
      "Group 3a, step 9: 0.9911449752883031\n",
      "Group 3b, step 1: 0.9880560131795717\n",
      "Group 3b, step 2: 0.9847611202635914\n",
      "Group 3b, step 3: 0.32434102141680393\n",
      "Group 3b, step 4: 0.3241350906095552\n",
      "Group 3b, step 5: 0.3239291598023064\n",
      "Group 3b, step 6: 0.20881383855024713\n",
      "Group 3b, step 7: 0.08422570016474465\n",
      "Group 3b, step 8: 0.08401976935749589\n",
      "Group 3b, step 9: 0.08401976935749589\n",
      "Group 3c, step 1: 0.9888797364085667\n",
      "Group 3c, step 2: 0.9880560131795717\n",
      "Group 3c, step 3: 0.32434102141680393\n",
      "Group 3c, step 4: 0.32434102141680393\n",
      "Group 3c, step 5: 0.3241350906095552\n",
      "Group 3c, step 6: 0.30004118616144976\n",
      "Group 3c, step 7: 0.30004118616144976\n",
      "Group 3c, step 8: 0.20922570016474465\n",
      "Group 3c, step 9: 0.20963756177924217\n",
      "Group 3d, step 1: 0.9969110378912686\n",
      "Group 3d, step 2: 0.996499176276771\n",
      "Group 3d, step 3: 0.9952635914332785\n",
      "Group 3d, step 4: 0.9921746293245469\n",
      "Group 3d, step 5: 0.8735584843492586\n",
      "Group 3d, step 6: 0.8607907742998353\n",
      "Group 3d, step 7: 0.8496705107084019\n",
      "Group 3d, step 8: 0.8496705107084019\n",
      "Group 3d, step 9: 0.8507001647446458\n",
      "Group 4a, step 1: 0.9880560131795717\n",
      "Group 4a, step 2: 0.9847611202635914\n",
      "Group 4a, step 3: 0.3239291598023064\n",
      "Group 4a, step 4: 0.30004118616144976\n",
      "Group 4a, step 5: 0.07866556836902801\n",
      "Group 4a, step 6: 0.07866556836902801\n",
      "Group 4a, step 7: 0.042627677100494234\n",
      "Group 4a, step 8: 0.04242174629324547\n",
      "Group 4a, step 9: 0.04242174629324547\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1a</th>\n",
       "      <td>0.996705</td>\n",
       "      <td>0.995881</td>\n",
       "      <td>0.995881</td>\n",
       "      <td>0.995881</td>\n",
       "      <td>0.995881</td>\n",
       "      <td>0.995881</td>\n",
       "      <td>0.995881</td>\n",
       "      <td>0.995675</td>\n",
       "      <td>0.995675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1b</th>\n",
       "      <td>0.997529</td>\n",
       "      <td>0.997529</td>\n",
       "      <td>0.996911</td>\n",
       "      <td>0.996911</td>\n",
       "      <td>0.996293</td>\n",
       "      <td>0.995470</td>\n",
       "      <td>0.995470</td>\n",
       "      <td>0.994852</td>\n",
       "      <td>0.994852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1c</th>\n",
       "      <td>0.997529</td>\n",
       "      <td>0.997529</td>\n",
       "      <td>0.997529</td>\n",
       "      <td>0.997529</td>\n",
       "      <td>0.997323</td>\n",
       "      <td>0.997323</td>\n",
       "      <td>0.997117</td>\n",
       "      <td>0.996911</td>\n",
       "      <td>0.996911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1d</th>\n",
       "      <td>0.997529</td>\n",
       "      <td>0.997323</td>\n",
       "      <td>0.996911</td>\n",
       "      <td>0.996705</td>\n",
       "      <td>0.996499</td>\n",
       "      <td>0.996499</td>\n",
       "      <td>0.996499</td>\n",
       "      <td>0.996499</td>\n",
       "      <td>0.996499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2a</th>\n",
       "      <td>0.996705</td>\n",
       "      <td>0.995675</td>\n",
       "      <td>0.995675</td>\n",
       "      <td>0.995470</td>\n",
       "      <td>0.995058</td>\n",
       "      <td>0.991557</td>\n",
       "      <td>0.991557</td>\n",
       "      <td>0.991351</td>\n",
       "      <td>0.991145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2b</th>\n",
       "      <td>0.996705</td>\n",
       "      <td>0.995881</td>\n",
       "      <td>0.995881</td>\n",
       "      <td>0.995675</td>\n",
       "      <td>0.995470</td>\n",
       "      <td>0.995264</td>\n",
       "      <td>0.995058</td>\n",
       "      <td>0.995058</td>\n",
       "      <td>0.995058</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2c</th>\n",
       "      <td>0.996499</td>\n",
       "      <td>0.988056</td>\n",
       "      <td>0.327842</td>\n",
       "      <td>0.327842</td>\n",
       "      <td>0.327842</td>\n",
       "      <td>0.327842</td>\n",
       "      <td>0.327842</td>\n",
       "      <td>0.327636</td>\n",
       "      <td>0.364498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2d</th>\n",
       "      <td>0.996911</td>\n",
       "      <td>0.996499</td>\n",
       "      <td>0.995264</td>\n",
       "      <td>0.995058</td>\n",
       "      <td>0.994852</td>\n",
       "      <td>0.912479</td>\n",
       "      <td>0.912273</td>\n",
       "      <td>0.911862</td>\n",
       "      <td>0.934926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2e</th>\n",
       "      <td>0.997529</td>\n",
       "      <td>0.997529</td>\n",
       "      <td>0.996911</td>\n",
       "      <td>0.996293</td>\n",
       "      <td>0.995470</td>\n",
       "      <td>0.995058</td>\n",
       "      <td>0.994852</td>\n",
       "      <td>0.994646</td>\n",
       "      <td>0.994234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2f</th>\n",
       "      <td>0.996911</td>\n",
       "      <td>0.996499</td>\n",
       "      <td>0.996499</td>\n",
       "      <td>0.996087</td>\n",
       "      <td>0.995058</td>\n",
       "      <td>0.994852</td>\n",
       "      <td>0.994852</td>\n",
       "      <td>0.960873</td>\n",
       "      <td>0.960873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3a</th>\n",
       "      <td>0.995881</td>\n",
       "      <td>0.995675</td>\n",
       "      <td>0.995470</td>\n",
       "      <td>0.995470</td>\n",
       "      <td>0.991557</td>\n",
       "      <td>0.991351</td>\n",
       "      <td>0.991351</td>\n",
       "      <td>0.991145</td>\n",
       "      <td>0.991145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3b</th>\n",
       "      <td>0.988056</td>\n",
       "      <td>0.984761</td>\n",
       "      <td>0.324341</td>\n",
       "      <td>0.324135</td>\n",
       "      <td>0.323929</td>\n",
       "      <td>0.208814</td>\n",
       "      <td>0.084226</td>\n",
       "      <td>0.084020</td>\n",
       "      <td>0.084020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3c</th>\n",
       "      <td>0.988880</td>\n",
       "      <td>0.988056</td>\n",
       "      <td>0.324341</td>\n",
       "      <td>0.324341</td>\n",
       "      <td>0.324135</td>\n",
       "      <td>0.300041</td>\n",
       "      <td>0.300041</td>\n",
       "      <td>0.209226</td>\n",
       "      <td>0.209638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3d</th>\n",
       "      <td>0.996911</td>\n",
       "      <td>0.996499</td>\n",
       "      <td>0.995264</td>\n",
       "      <td>0.992175</td>\n",
       "      <td>0.873558</td>\n",
       "      <td>0.860791</td>\n",
       "      <td>0.849671</td>\n",
       "      <td>0.849671</td>\n",
       "      <td>0.850700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4a</th>\n",
       "      <td>0.988056</td>\n",
       "      <td>0.984761</td>\n",
       "      <td>0.323929</td>\n",
       "      <td>0.300041</td>\n",
       "      <td>0.078666</td>\n",
       "      <td>0.078666</td>\n",
       "      <td>0.042628</td>\n",
       "      <td>0.042422</td>\n",
       "      <td>0.042422</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2         3         4         5         6  \\\n",
       "1a  0.996705  0.995881  0.995881  0.995881  0.995881  0.995881  0.995881   \n",
       "1b  0.997529  0.997529  0.996911  0.996911  0.996293  0.995470  0.995470   \n",
       "1c  0.997529  0.997529  0.997529  0.997529  0.997323  0.997323  0.997117   \n",
       "1d  0.997529  0.997323  0.996911  0.996705  0.996499  0.996499  0.996499   \n",
       "2a  0.996705  0.995675  0.995675  0.995470  0.995058  0.991557  0.991557   \n",
       "2b  0.996705  0.995881  0.995881  0.995675  0.995470  0.995264  0.995058   \n",
       "2c  0.996499  0.988056  0.327842  0.327842  0.327842  0.327842  0.327842   \n",
       "2d  0.996911  0.996499  0.995264  0.995058  0.994852  0.912479  0.912273   \n",
       "2e  0.997529  0.997529  0.996911  0.996293  0.995470  0.995058  0.994852   \n",
       "2f  0.996911  0.996499  0.996499  0.996087  0.995058  0.994852  0.994852   \n",
       "3a  0.995881  0.995675  0.995470  0.995470  0.991557  0.991351  0.991351   \n",
       "3b  0.988056  0.984761  0.324341  0.324135  0.323929  0.208814  0.084226   \n",
       "3c  0.988880  0.988056  0.324341  0.324341  0.324135  0.300041  0.300041   \n",
       "3d  0.996911  0.996499  0.995264  0.992175  0.873558  0.860791  0.849671   \n",
       "4a  0.988056  0.984761  0.323929  0.300041  0.078666  0.078666  0.042628   \n",
       "\n",
       "           7         8  \n",
       "1a  0.995675  0.995675  \n",
       "1b  0.994852  0.994852  \n",
       "1c  0.996911  0.996911  \n",
       "1d  0.996499  0.996499  \n",
       "2a  0.991351  0.991145  \n",
       "2b  0.995058  0.995058  \n",
       "2c  0.327636  0.364498  \n",
       "2d  0.911862  0.934926  \n",
       "2e  0.994646  0.994234  \n",
       "2f  0.960873  0.960873  \n",
       "3a  0.991145  0.991145  \n",
       "3b  0.084020  0.084020  \n",
       "3c  0.209226  0.209638  \n",
       "3d  0.849671  0.850700  \n",
       "4a  0.042422  0.042422  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.998383</td>\n",
       "      <td>0.995935</td>\n",
       "      <td>0.997158</td>\n",
       "      <td>93000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.922509</td>\n",
       "      <td>0.967742</td>\n",
       "      <td>0.944584</td>\n",
       "      <td>4650.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.994593</td>\n",
       "      <td>0.994593</td>\n",
       "      <td>0.994593</td>\n",
       "      <td>0.994593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>macro avg</th>\n",
       "      <td>0.960446</td>\n",
       "      <td>0.981839</td>\n",
       "      <td>0.970871</td>\n",
       "      <td>97650.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weighted avg</th>\n",
       "      <td>0.994770</td>\n",
       "      <td>0.994593</td>\n",
       "      <td>0.994654</td>\n",
       "      <td>97650.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              precision    recall  f1-score       support\n",
       "0              0.998383  0.995935  0.997158  93000.000000\n",
       "1              0.922509  0.967742  0.944584   4650.000000\n",
       "accuracy       0.994593  0.994593  0.994593      0.994593\n",
       "macro avg      0.960446  0.981839  0.970871  97650.000000\n",
       "weighted avg   0.994770  0.994593  0.994654  97650.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f1b5d4629b34b84a8cd50328dca0bf4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/135 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Group 1a, step 1: 0.9759139784946237\n",
      "Group 1a, step 2: 0.9701075268817204\n",
      "Group 1a, step 3: 0.9696774193548388\n",
      "Group 1a, step 4: 0.9696774193548388\n",
      "Group 1a, step 5: 0.9696774193548388\n",
      "Group 1a, step 6: 0.9705376344086022\n",
      "Group 1a, step 7: 0.970752688172043\n",
      "Group 1a, step 8: 0.9709677419354839\n",
      "Group 1a, step 9: 0.9765591397849462\n",
      "Group 1b, step 1: 0.9664516129032258\n",
      "Group 1b, step 2: 0.9664516129032258\n",
      "Group 1b, step 3: 0.9653763440860215\n",
      "Group 1b, step 4: 0.9311827956989247\n",
      "Group 1b, step 5: 0.930752688172043\n",
      "Group 1b, step 6: 0.9305376344086022\n",
      "Group 1b, step 7: 0.9653763440860215\n",
      "Group 1b, step 8: 0.9662365591397849\n",
      "Group 1b, step 9: 0.9688172043010753\n",
      "Group 1c, step 1: 0.9673118279569892\n",
      "Group 1c, step 2: 0.9673118279569892\n",
      "Group 1c, step 3: 0.9666666666666667\n",
      "Group 1c, step 4: 0.9666666666666667\n",
      "Group 1c, step 5: 0.9662365591397849\n",
      "Group 1c, step 6: 0.9662365591397849\n",
      "Group 1c, step 7: 0.9666666666666667\n",
      "Group 1c, step 8: 0.9666666666666667\n",
      "Group 1c, step 9: 0.9673118279569892\n",
      "Group 1d, step 1: 0.967741935483871\n",
      "Group 1d, step 2: 0.9670967741935483\n",
      "Group 1d, step 3: 0.9316129032258065\n",
      "Group 1d, step 4: 0.9264516129032258\n",
      "Group 1d, step 5: 0.9232258064516129\n",
      "Group 1d, step 6: 0.9202150537634408\n",
      "Group 1d, step 7: 0.9184946236559139\n",
      "Group 1d, step 8: 0.9129032258064517\n",
      "Group 1d, step 9: 0.9118279569892473\n",
      "Group 2a, step 1: 0.9746236559139785\n",
      "Group 2a, step 2: 0.9675268817204301\n",
      "Group 2a, step 3: 0.9331182795698925\n",
      "Group 2a, step 4: 0.9329032258064516\n",
      "Group 2a, step 5: 0.9329032258064516\n",
      "Group 2a, step 6: 0.9331182795698925\n",
      "Group 2a, step 7: 0.9675268817204301\n",
      "Group 2a, step 8: 0.9688172043010753\n",
      "Group 2a, step 9: 0.9776344086021506\n",
      "Group 2b, step 1: 0.9754838709677419\n",
      "Group 2b, step 2: 0.9694623655913979\n",
      "Group 2b, step 3: 0.9341935483870968\n",
      "Group 2b, step 4: 0.9335483870967742\n",
      "Group 2b, step 5: 0.9335483870967742\n",
      "Group 2b, step 6: 0.9339784946236559\n",
      "Group 2b, step 7: 0.9350537634408602\n",
      "Group 2b, step 8: 0.9356989247311828\n",
      "Group 2b, step 9: 0.9709677419354839\n",
      "Group 2c, step 1: 0.970752688172043\n",
      "Group 2c, step 2: 0.9683870967741935\n",
      "Group 2c, step 3: 0.9333333333333333\n",
      "Group 2c, step 4: 0.9283870967741935\n",
      "Group 2c, step 5: 0.6739784946236559\n",
      "Group 2c, step 6: 0.6701075268817205\n",
      "Group 2c, step 7: 0.6698924731182796\n",
      "Group 2c, step 8: 0.6804301075268817\n",
      "Group 2c, step 9: 0.6853763440860215\n",
      "Group 2d, step 1: 0.9662365591397849\n",
      "Group 2d, step 2: 0.9634408602150538\n",
      "Group 2d, step 3: 0.9124731182795699\n",
      "Group 2d, step 4: 0.8898924731182796\n",
      "Group 2d, step 5: 0.8836559139784946\n",
      "Group 2d, step 6: 0.8761290322580645\n",
      "Group 2d, step 7: 0.8894623655913978\n",
      "Group 2d, step 8: 0.9\n",
      "Group 2d, step 9: 0.9116129032258065\n",
      "Group 2e, step 1: 0.9664516129032258\n",
      "Group 2e, step 2: 0.9662365591397849\n",
      "Group 2e, step 3: 0.9313978494623656\n",
      "Group 2e, step 4: 0.930752688172043\n",
      "Group 2e, step 5: 0.9303225806451613\n",
      "Group 2e, step 6: 0.9303225806451613\n",
      "Group 2e, step 7: 0.9655913978494624\n",
      "Group 2e, step 8: 0.9673118279569892\n",
      "Group 2e, step 9: 0.9688172043010753\n",
      "Group 2f, step 1: 0.9670967741935483\n",
      "Group 2f, step 2: 0.9660215053763441\n",
      "Group 2f, step 3: 0.9260215053763441\n",
      "Group 2f, step 4: 0.9133333333333333\n",
      "Group 2f, step 5: 0.9079569892473118\n",
      "Group 2f, step 6: 0.8984946236559139\n",
      "Group 2f, step 7: 0.9004301075268817\n",
      "Group 2f, step 8: 0.8984946236559139\n",
      "Group 2f, step 9: 0.8941935483870967\n",
      "Group 3a, step 1: 0.9744086021505376\n",
      "Group 3a, step 2: 0.9673118279569892\n",
      "Group 3a, step 3: 0.9329032258064516\n",
      "Group 3a, step 4: 0.9320430107526881\n",
      "Group 3a, step 5: 0.9320430107526881\n",
      "Group 3a, step 6: 0.9324731182795699\n",
      "Group 3a, step 7: 0.9335483870967742\n",
      "Group 3a, step 8: 0.9698924731182795\n",
      "Group 3a, step 9: 0.9774193548387097\n",
      "Group 3b, step 1: 0.9651612903225807\n",
      "Group 3b, step 2: 0.9587096774193549\n",
      "Group 3b, step 3: 0.6346236559139785\n",
      "Group 3b, step 4: 0.6298924731182796\n",
      "Group 3b, step 5: 0.6227956989247312\n",
      "Group 3b, step 6: 0.6161290322580645\n",
      "Group 3b, step 7: 0.8778494623655914\n",
      "Group 3b, step 8: 0.8870967741935484\n",
      "Group 3b, step 9: 0.9116129032258065\n",
      "Group 3c, step 1: 0.9688172043010753\n",
      "Group 3c, step 2: 0.9649462365591398\n",
      "Group 3c, step 3: 0.6890322580645162\n",
      "Group 3c, step 4: 0.9058064516129032\n",
      "Group 3c, step 5: 0.6281720430107527\n",
      "Group 3c, step 6: 0.6341935483870967\n",
      "Group 3c, step 7: 0.6393548387096775\n",
      "Group 3c, step 8: 0.6421505376344085\n",
      "Group 3c, step 9: 0.8886021505376344\n",
      "Group 3d, step 1: 0.9662365591397849\n",
      "Group 3d, step 2: 0.9625806451612903\n",
      "Group 3d, step 3: 0.9096774193548387\n",
      "Group 3d, step 4: 0.886021505376344\n",
      "Group 3d, step 5: 0.8821505376344086\n",
      "Group 3d, step 6: 0.8767741935483871\n",
      "Group 3d, step 7: 0.8898924731182796\n",
      "Group 3d, step 8: 0.8933333333333333\n",
      "Group 3d, step 9: 0.8950537634408602\n",
      "Group 4a, step 1: 0.9651612903225807\n",
      "Group 4a, step 2: 0.9587096774193549\n",
      "Group 4a, step 3: 0.6346236559139785\n",
      "Group 4a, step 4: 0.6298924731182796\n",
      "Group 4a, step 5: 0.6193548387096774\n",
      "Group 4a, step 6: 0.613978494623656\n",
      "Group 4a, step 7: 0.8797849462365591\n",
      "Group 4a, step 8: 0.8937634408602151\n",
      "Group 4a, step 9: 0.9124731182795699\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1a</th>\n",
       "      <td>0.975914</td>\n",
       "      <td>0.970108</td>\n",
       "      <td>0.969677</td>\n",
       "      <td>0.969677</td>\n",
       "      <td>0.969677</td>\n",
       "      <td>0.970538</td>\n",
       "      <td>0.970753</td>\n",
       "      <td>0.970968</td>\n",
       "      <td>0.976559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1b</th>\n",
       "      <td>0.966452</td>\n",
       "      <td>0.966452</td>\n",
       "      <td>0.965376</td>\n",
       "      <td>0.931183</td>\n",
       "      <td>0.930753</td>\n",
       "      <td>0.930538</td>\n",
       "      <td>0.965376</td>\n",
       "      <td>0.966237</td>\n",
       "      <td>0.968817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1c</th>\n",
       "      <td>0.967312</td>\n",
       "      <td>0.967312</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>0.966237</td>\n",
       "      <td>0.966237</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>0.967312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1d</th>\n",
       "      <td>0.967742</td>\n",
       "      <td>0.967097</td>\n",
       "      <td>0.931613</td>\n",
       "      <td>0.926452</td>\n",
       "      <td>0.923226</td>\n",
       "      <td>0.920215</td>\n",
       "      <td>0.918495</td>\n",
       "      <td>0.912903</td>\n",
       "      <td>0.911828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2a</th>\n",
       "      <td>0.974624</td>\n",
       "      <td>0.967527</td>\n",
       "      <td>0.933118</td>\n",
       "      <td>0.932903</td>\n",
       "      <td>0.932903</td>\n",
       "      <td>0.933118</td>\n",
       "      <td>0.967527</td>\n",
       "      <td>0.968817</td>\n",
       "      <td>0.977634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2b</th>\n",
       "      <td>0.975484</td>\n",
       "      <td>0.969462</td>\n",
       "      <td>0.934194</td>\n",
       "      <td>0.933548</td>\n",
       "      <td>0.933548</td>\n",
       "      <td>0.933978</td>\n",
       "      <td>0.935054</td>\n",
       "      <td>0.935699</td>\n",
       "      <td>0.970968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2c</th>\n",
       "      <td>0.970753</td>\n",
       "      <td>0.968387</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>0.928387</td>\n",
       "      <td>0.673978</td>\n",
       "      <td>0.670108</td>\n",
       "      <td>0.669892</td>\n",
       "      <td>0.680430</td>\n",
       "      <td>0.685376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2d</th>\n",
       "      <td>0.966237</td>\n",
       "      <td>0.963441</td>\n",
       "      <td>0.912473</td>\n",
       "      <td>0.889892</td>\n",
       "      <td>0.883656</td>\n",
       "      <td>0.876129</td>\n",
       "      <td>0.889462</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.911613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2e</th>\n",
       "      <td>0.966452</td>\n",
       "      <td>0.966237</td>\n",
       "      <td>0.931398</td>\n",
       "      <td>0.930753</td>\n",
       "      <td>0.930323</td>\n",
       "      <td>0.930323</td>\n",
       "      <td>0.965591</td>\n",
       "      <td>0.967312</td>\n",
       "      <td>0.968817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2f</th>\n",
       "      <td>0.967097</td>\n",
       "      <td>0.966022</td>\n",
       "      <td>0.926022</td>\n",
       "      <td>0.913333</td>\n",
       "      <td>0.907957</td>\n",
       "      <td>0.898495</td>\n",
       "      <td>0.900430</td>\n",
       "      <td>0.898495</td>\n",
       "      <td>0.894194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3a</th>\n",
       "      <td>0.974409</td>\n",
       "      <td>0.967312</td>\n",
       "      <td>0.932903</td>\n",
       "      <td>0.932043</td>\n",
       "      <td>0.932043</td>\n",
       "      <td>0.932473</td>\n",
       "      <td>0.933548</td>\n",
       "      <td>0.969892</td>\n",
       "      <td>0.977419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3b</th>\n",
       "      <td>0.965161</td>\n",
       "      <td>0.958710</td>\n",
       "      <td>0.634624</td>\n",
       "      <td>0.629892</td>\n",
       "      <td>0.622796</td>\n",
       "      <td>0.616129</td>\n",
       "      <td>0.877849</td>\n",
       "      <td>0.887097</td>\n",
       "      <td>0.911613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3c</th>\n",
       "      <td>0.968817</td>\n",
       "      <td>0.964946</td>\n",
       "      <td>0.689032</td>\n",
       "      <td>0.905806</td>\n",
       "      <td>0.628172</td>\n",
       "      <td>0.634194</td>\n",
       "      <td>0.639355</td>\n",
       "      <td>0.642151</td>\n",
       "      <td>0.888602</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3d</th>\n",
       "      <td>0.966237</td>\n",
       "      <td>0.962581</td>\n",
       "      <td>0.909677</td>\n",
       "      <td>0.886022</td>\n",
       "      <td>0.882151</td>\n",
       "      <td>0.876774</td>\n",
       "      <td>0.889892</td>\n",
       "      <td>0.893333</td>\n",
       "      <td>0.895054</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4a</th>\n",
       "      <td>0.965161</td>\n",
       "      <td>0.958710</td>\n",
       "      <td>0.634624</td>\n",
       "      <td>0.629892</td>\n",
       "      <td>0.619355</td>\n",
       "      <td>0.613978</td>\n",
       "      <td>0.879785</td>\n",
       "      <td>0.893763</td>\n",
       "      <td>0.912473</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2         3         4         5         6  \\\n",
       "1a  0.975914  0.970108  0.969677  0.969677  0.969677  0.970538  0.970753   \n",
       "1b  0.966452  0.966452  0.965376  0.931183  0.930753  0.930538  0.965376   \n",
       "1c  0.967312  0.967312  0.966667  0.966667  0.966237  0.966237  0.966667   \n",
       "1d  0.967742  0.967097  0.931613  0.926452  0.923226  0.920215  0.918495   \n",
       "2a  0.974624  0.967527  0.933118  0.932903  0.932903  0.933118  0.967527   \n",
       "2b  0.975484  0.969462  0.934194  0.933548  0.933548  0.933978  0.935054   \n",
       "2c  0.970753  0.968387  0.933333  0.928387  0.673978  0.670108  0.669892   \n",
       "2d  0.966237  0.963441  0.912473  0.889892  0.883656  0.876129  0.889462   \n",
       "2e  0.966452  0.966237  0.931398  0.930753  0.930323  0.930323  0.965591   \n",
       "2f  0.967097  0.966022  0.926022  0.913333  0.907957  0.898495  0.900430   \n",
       "3a  0.974409  0.967312  0.932903  0.932043  0.932043  0.932473  0.933548   \n",
       "3b  0.965161  0.958710  0.634624  0.629892  0.622796  0.616129  0.877849   \n",
       "3c  0.968817  0.964946  0.689032  0.905806  0.628172  0.634194  0.639355   \n",
       "3d  0.966237  0.962581  0.909677  0.886022  0.882151  0.876774  0.889892   \n",
       "4a  0.965161  0.958710  0.634624  0.629892  0.619355  0.613978  0.879785   \n",
       "\n",
       "           7         8  \n",
       "1a  0.970968  0.976559  \n",
       "1b  0.966237  0.968817  \n",
       "1c  0.966667  0.967312  \n",
       "1d  0.912903  0.911828  \n",
       "2a  0.968817  0.977634  \n",
       "2b  0.935699  0.970968  \n",
       "2c  0.680430  0.685376  \n",
       "2d  0.900000  0.911613  \n",
       "2e  0.967312  0.968817  \n",
       "2f  0.898495  0.894194  \n",
       "3a  0.969892  0.977419  \n",
       "3b  0.887097  0.911613  \n",
       "3c  0.642151  0.888602  \n",
       "3d  0.893333  0.895054  \n",
       "4a  0.893763  0.912473  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if not os.path.exists('results/ToN_IoT/scores/rf/'):\n",
    "    os.mkdir('results/ToN_IoT/scores/rf/')\n",
    "\n",
    "for attack in ['ddos', 'backdoor', 'dos', 'injection', 'password', 'ransomware', 'scanning', 'xss']:\n",
    "\n",
    "    if not os.path.exists(f'results/ToN_IoT/scores/rf/{attack}'):\n",
    "        os.mkdir(f'results/ToN_IoT/scores/rf/{attack}')\n",
    "    \n",
    "    train_data = pd.read_csv(f'preprocessed_data/ToN_IoT/{attack}_train.csv')\n",
    "    test_data = pd.read_csv(f'preprocessed_data/ToN_IoT/{attack}_test.csv')\n",
    "    train_data.drop(columns=['Src IP', 'Dst IP'], inplace=True)\n",
    "    test_data.drop(columns=['Src IP', 'Dst IP'], inplace=True)\n",
    "    \n",
    "    model = RandomForestClassifier(n_estimators=100, n_jobs=-1)\n",
    "    model.fit(train_data.drop(columns=['Label']), train_data.Label)\n",
    "    baseline_pred = model.predict(test_data.drop(columns=['Label']))\n",
    "    report = classification_report(test_data.Label, baseline_pred, digits=3, output_dict=True)\n",
    "    report = pd.DataFrame(report).transpose()\n",
    "    display(report)\n",
    "    \n",
    "    report.to_csv(f'results/ToN_IoT/scores/rf/{attack}/baseline.csv')\n",
    "    \n",
    "    recall, precision, f1 = adversarial_feature_attack(test_data, model)\n",
    "    display(recall)\n",
    "    \n",
    "    recall.to_csv(f'results/ToN_IoT/scores/rf/{attack}/feature_attack.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
